From 37a7d4e83276ff1471a28ff41f512d4ad020a292 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Wed, 5 Aug 2026 08:46:47 +0800 Subject: [PATCH 001/465] Align vLLM-Omni v0.26.0 knowledge baseline --- adapters/vllm_omni/manifest.yaml | 6 +-- adapters/vllm_omni/release_baseline.yaml | 28 +++++----- doc/architecture/KNOWLEDGE.md | 4 +- .../ci/guides/inspect-existing-tests-first.md | 28 +++++----- .../benchmark/guides/performance-evidence.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 2 +- .../ci/guides/buildkite-structure.md | 35 +++++++------ .../repos/vllm-omni/ci/guides/test-tiers.md | 4 +- .../components/configuration/_index.md | 7 +-- .../components/configuration/architecture.md | 6 +-- .../components/configuration/deploy-yaml.md | 9 ++-- .../configuration/omni-init-args.md | 4 +- .../configuration/pipeline-deploy-schema.md | 13 ++--- .../components/configuration/rules.md | 40 +++++++++++++- .../vllm-omni/components/diffusion/_index.md | 3 +- .../vllm-omni/components/diffusion/rules.md | 33 ++++++++++-- .../components/distributed/_index.md | 5 +- .../vllm-omni/components/distributed/rules.md | 36 +++++++++++++ .../components/model-executor/_index.md | 2 +- .../components/model-executor/architecture.md | 6 +-- .../components/model-executor/rules.md | 40 +++++++++++++- .../vllm-omni/components/scheduler/_index.md | 4 +- .../components/scheduler/architecture.md | 13 +++-- .../vllm-omni/components/scheduler/rules.md | 52 ++++++++++++++++++- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules.md | 40 +++++++++++++- .../repos/vllm-omni/docs/design-doc-map.md | 4 +- .../repos/vllm-omni/models/audex/_index.md | 51 ++++++++++++++++++ knowledge/repos/vllm-omni/models/catalog.md | 42 ++++++++------- .../models/dreamzero/architecture.md | 8 +-- .../vllm-omni/models/gr00t/architecture.md | 2 +- .../models/higgs-audio/architecture.md | 2 +- .../vllm-omni/models/hunyuan-video/_index.md | 2 +- .../repos/vllm-omni/models/ltx2/_index.md | 16 +++--- .../vllm-omni/models/ltx2/architecture.md | 21 ++++---- .../models/mammoth-moda2/architecture.md | 2 +- .../vllm-omni/models/minimax-h3/_index.md | 40 ++++++++++++++ .../vllm-omni/models/soulx-singer/_index.md | 2 +- .../models/soulx-singer/architecture.md | 2 +- knowledge/repos/vllm-omni/rebase/_index.md | 6 +-- .../vllm-omni/rebase/upstream-api-drift.md | 7 +-- knowledge/repos/vllm-omni/rebase/workflow.md | 8 +-- 42 files changed, 488 insertions(+), 151 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/distributed/rules.md create mode 100644 knowledge/repos/vllm-omni/models/audex/_index.md create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/_index.md diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index 4be71bd2f..d530dd2bd 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -72,7 +72,7 @@ modules: wave: 1 risk: high model_executor: - local_paths: [vllm_omni/model_executor/] + local_paths: [vllm_omni/model_executor/, vllm_omni/experimental/, vllm_omni/model_extras/] upstream_paths: [vllm/model_executor/] test_paths: [tests/model_executor/, tests/diffusion/, tests/e2e/offline_inference/] import_check: "from vllm_omni.model_executor.models.registry import *; print('Model Executor imports OK')" @@ -92,13 +92,13 @@ modules: wave: 1 risk: high online_serving: - local_paths: [vllm_omni/entrypoints/] + local_paths: [vllm_omni/entrypoints/, vllm_omni/engine/] upstream_paths: [vllm/entrypoints/] test_paths: [tests/entrypoints/test_async_omni.py, tests/entrypoints/test_async_omni_diffusion_config.py, tests/entrypoints/openai_api/] import_check: "from vllm_omni.entrypoints.omni import Omni; from vllm_omni.entrypoints.async_omni import AsyncOmni; from vllm_omni.entrypoints.openai.api_server import omni_run_server; from vllm_omni.entrypoints.openai.serving_speech import ServingSpeech; from vllm_omni.entrypoints.cli.serve import OmniServeCommand; print('Online Serving imports OK')" wave: 1 model_config: - local_paths: [vllm_omni/config/, vllm_omni/engine/arg_utils.py] + local_paths: [vllm_omni/config/, vllm_omni/deploy/, vllm_omni/engine/arg_utils.py] upstream_paths: [vllm/config/, vllm/engine/arg_utils.py] test_paths: [tests/entrypoints/test_stage_utils.py, tests/metrics/test_stats.py] import_check: "from vllm_omni.config.model import OmniModelConfig; from vllm_omni.engine.arg_utils import OmniEngineArgs; print('Model Config imports OK')" diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d7e0dc30f..a6094762f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5d44868e918ecf9d3a6f1158c45acfad5989e1a7 - audited_ref: v0.26.0rc1 - audited_sha: 807db6efd70ff2e9b55a63d6e1b0530e2b74f8f2 + previous_audited_sha: 807db6efd70ff2e9b55a63d6e1b0530e2b74f8f2 + audited_ref: v0.26.0 + audited_sha: a4ea67a21b20054dacc6e83952f9bd407e8ee4e7 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 72 - sha256: 79afac742125003b8d65a02e0e6b53398c4fc5f1c6e765fa8e1eb31f74871195 + count: 77 + sha256: 10e7850b0c7804f23ddffeb7cf5f2ad83c616d3977ec3b67eabab5c121564a49 diffusion: - count: 61 - sha256: f5c0b782cb98f20a9c7c77b696fda620a12d0899fd9e8abc9ccb696fe0c457e1 + count: 58 + sha256: 7dc6a5a373ca33a31a44961c1422008327cede3b364577fa095fd37c05ea2085 pipelines: - count: 46 - sha256: ce4f6edb8748e030f04bf7a6fe038231453cd1cd75f64311c898acff8c14b14b + count: 51 + sha256: 8f567368a2a958fee2f89e4bce3ec20ec2e7a420bbdbab2fef9d65fb604e3ae8 deploy_yamls: - count: 71 - sha256: fb4c591580262a2dc8d03dd8c7fd681d98486f31007696d56adf349c24c25147 + count: 79 + sha256: fb75633eeeaadc4f421fd4d2ec62b03b04d9a9f48e959297d88ae51c20d51bc5 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. @@ -44,7 +44,6 @@ path_owners: - docs/ - examples/ - recipes/ - - realtime_video_prompt_interaction_protocol.md model-executor: - vllm_omni/attention/ - vllm_omni/experimental/ @@ -74,6 +73,7 @@ path_owners: - tests/ tooling: - tools/ + - apps/ComfyUI-vLLM-Omni/ runtime-core: - vllm_omni/__init__.py - vllm_omni/data_entry_keys.py @@ -91,7 +91,7 @@ owner_documents: diffusion: - knowledge/repos/vllm-omni/components/diffusion/rules.md distributed: - - knowledge/repos/vllm-omni/components/distributed/_index.md + - knowledge/repos/vllm-omni/components/distributed/rules.md documentation: - knowledge/general/docs/_index.md model-executor: @@ -124,6 +124,8 @@ ignored_paths: reason: legal metadata - pattern: README.md reason: repository overview +- pattern: realtime_video_prompt_interaction_protocol.md + reason: deleted upstream documentation artifact - pattern: SECURITY.md reason: community metadata - pattern: setup.py diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d5a0df6a3..21638ad6f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -41,8 +41,8 @@ extensions merely because they are absent from the common source. (per-file upstream created/updated dates, captured before the submodule's git history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps - are verified against vllm-omni `main @ - 807db6efd70ff2e9b55a63d6e1b0530e2b74f8f2`. The canonical machine baseline + are verified against vllm-omni `v0.26.0 @ + a4ea67a21b20054dacc6e83952f9bd407e8ee4e7`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/general/ci/guides/inspect-existing-tests-first.md b/knowledge/general/ci/guides/inspect-existing-tests-first.md index f77ed4d9b..b1bf22ae3 100644 --- a/knowledge/general/ci/guides/inspect-existing-tests-first.md +++ b/knowledge/general/ci/guides/inspect-existing-tests-first.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-07-16 type: guide tags: [general, ci] -sources: ["#3297", "tests/e2e/accuracy/test_gebench_h100_smoke.py", "tests/e2e/accuracy/test_hunyuan_image3.py"] +sources: ["#3297", "tests/e2e/accuracy/test_hunyuan_image3.py", "tests/e2e/accuracy/conftest.py"] --- # 规则：先看现有代码，再谈方案 @@ -16,13 +16,11 @@ sources: ["#3297", "tests/e2e/accuracy/test_gebench_h100_smoke.py", "tests/e2e/a - 讨论"用 CLIP-score 替代"、"改 GenBench 砍 it2i"、"采集 9 小时 baseline" - 写了 300 行方案对比文档 -结果用户直接去看仓库： -``` -tests/e2e/accuracy/test_gebench_h100_smoke.py -``` -**已经存在**，跑的就是 type3/type4 T2I，用 `/v1/images/generations`，judge 是 VLM-as-judge（Qwen2.5-VL-7B-Instruct），**零 mmdet/mmcv 依赖**。 - -`--gebench-model` 还是 CLI 参数。加一行 `--gebench-model Tencent/HunyuanImage-3.0-Instruct` 就能跑。 +结果用户直接去看当前 target 的 `tests/e2e/accuracy/`：旧记录中的 +`test_gebench_h100_smoke.py` 和 `test_gedit_bench_h100_smoke.py` 已经删除；当前 +HunyuanImage3 的入口是 `tests/e2e/accuracy/test_hunyuan_image3.py`，并由 +`tests/e2e/accuracy/conftest.py` 提供仍然存在的 accuracy fixtures。旧的 +`--gebench-model` 路径不能再当作现行合同。 我前面所有工作（踩 mmcv 坑、建 Py3.10 venv、写 GenEval 集成代码）全部白做。 @@ -45,23 +43,23 @@ tests/e2e/accuracy/test_gebench_h100_smoke.py | 目录 | 内容 | |---|---| | `tests/e2e/accuracy/` | 所有精度 CI 入口 | -| `tests/e2e/accuracy/conftest.py` | fixture 和 CLI option（`--gebench-model`, `--gedit-model`, `--accuracy-judge-model` 等） | -| `tests/e2e/accuracy/test_gebench_h100_smoke.py` | T2I 精度（type3/type4）用 VLM judge | -| `tests/e2e/accuracy/test_gedit_bench_h100_smoke.py` | IT2I 精度，同款 judge 路径 | +| `tests/e2e/accuracy/conftest.py` | 当前仍存在的 Wan2.2/HunyuanVideo accuracy fixtures 和 CLI options | +| `tests/e2e/accuracy/test_hunyuan_image3.py` | HunyuanImage3 IT2I/COT 对齐入口 | +| `tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py` | HunyuanImage3 像素精度入口 | | `tests/e2e/accuracy/helpers.py` | `reset_artifact_dir` 等 | -| `vllm_omni/benchmarks/accuracy/text_to_image/gbench.py` | GEBench 实现（type3/type4 走 `/v1/images/generations`，type1/2/5 走 `/v1/images/edits`） | ## 下一次做新模型精度 CI -抄 `test_gebench_h100_smoke.py`，改 `--gebench-model` 参数。完成。 -**除非有强理由**（比如新模型 T2I endpoint 不同、判据不同），否则不要考虑 GenEval / CLIP-score / 自研评分器。 +先从当前 `tests/e2e/accuracy/` 选择语义最接近的幸存者，读取它的 fixture、CLI +选项和运行方式，再决定复用还是新增；不要假设历史 GEBench 文件或参数仍存在。 ## 判据模板 用户问"做 XX 精度 CI"时，先回答 3 个问题再提方案： 1. 团队现有类似测试文件？叫什么名字？ 2. 它的 fixture 和 CLI option 我的模型能复用吗？ -3. 跑一下 `pytest --gebench-model ` 会发生什么？报错才说明需要改，没报错就结束了 +3. 跑一下当前幸存者的最小 pytest 入口会发生什么？只有现行路径的采集或运行错误 + 才说明需要改，不要为已删除的 slug 重建测试文件。 ## HunyuanImage3 现成 IT2I accuracy pytest（2026-06-02） diff --git a/knowledge/repos/vllm-omni/benchmark/guides/performance-evidence.md b/knowledge/repos/vllm-omni/benchmark/guides/performance-evidence.md index 28eccc60b..8a2e526a3 100644 --- a/knowledge/repos/vllm-omni/benchmark/guides/performance-evidence.md +++ b/knowledge/repos/vllm-omni/benchmark/guides/performance-evidence.md @@ -50,7 +50,7 @@ sources: ["PR #5052"] **PR Test Result 规范**： - 表格标题必须带输入口径，例如 `Official IT2I performance comparison`，不要只写 `Performance`。 -- 精度表必须写 reference，例如 `against tests/e2e/accuracy/assets/hunyuan_image_ref.png`。 +- 精度表必须写 reference，例如 `against tests/assets/hunyuan/hunyuan_image_ref.png`。 - 速度表必须写 `model initialization excluded/included`。 - 如果同一 PR 同时有 smoke 和 official e2e，smoke 放在最后，且标题写 `Compatibility smoke`。 diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index b4cc07af3..dfde01cde 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-08-23 type: index tags: [vllm-omni, ci] -sources: [] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md] --- # vLLM-Omni CI diff --git a/knowledge/repos/vllm-omni/ci/guides/buildkite-structure.md b/knowledge/repos/vllm-omni/ci/guides/buildkite-structure.md index 9ea7ff7ba..0799436b6 100644 --- a/knowledge/repos/vllm-omni/ci/guides/buildkite-structure.md +++ b/knowledge/repos/vllm-omni/ci/guides/buildkite-structure.md @@ -1,37 +1,40 @@ --- title: "Buildkite 管线结构" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-08-05 type: guide tags: [vllm-omni, ci] -sources: [".buildkite/pipeline.yml", "vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh"] +sources: [".buildkite/cuda/pipeline.yml", ".buildkite/cuda/rebase-pipeline.yml", "vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh"] --- # Buildkite 管线结构 -仓库侧事实在 `main @ 5c390096` 复核；运营侧事实（管线名、队列→硬件映射）来自 +仓库侧事实在 `v0.26.0 @ a4ea67a2` 复核；运营侧事实（管线名、队列→硬件映射）来自 rebase-agent 配置快照（@122a9468），**属运营观测、可能漂移**，用前核对。 ## `.buildkite/` 布局（仓库侧） -- **`pipeline.yml`**（根入口，two-doc 模式）：文档 1 经 - `.buildkite/scripts/upload_pipeline.py` 做 skip-ci 判定（docs-only / pytest - skip 标记，diff-aware）；`---` 后的文档 2 构建 CI 镜像（`docker/Dockerfile.ci` → - `public.ecr.aws/q9t5s3a7/vllm-ci-test-repo`）并按条件上载子管线。 +- **`cuda/pipeline.yml`**（CUDA hook entry）：由 + `.buildkite/common/scripts/upload_pipeline.py` 读取 `cuda/bootstrap-upload-steps.yml` + 并上传实际子管线；skip-ci 的 step key 是 `upload-ci-pipeline`，不再依赖根目录 + `pipeline.yml` 的 two-doc 结构。 - 分级 → 子管线映射：**L2 → `test-ready.yml`**（带 `ready` label 的 PR，diff-aware； 或 main+nightly）；**L3 → `test-merge.yml`**（`merge-test` label 或 main+nightly）； **L4 → `test-nightly.yml`**（main+nightly 定时，或 PR label + rebuild）； **L5 → `test-weekly.yml`**（每周，依赖镜像构建）。 -- 平台管线：`pipeline-intel.yaml`、`pipeline-npu.yaml`、`pipeline-npu-a3.yaml`； - AMD 变体 `test-amd.yaml`/`test-amd-ready.yaml`/`test-amd-merge.yml` + +- 平台管线：`intel/pipeline-intel.yml`、`npu/pipeline-npu.yml`、 + `npu/pipeline-npu-a3.yml`；AMD 变体 `amd/test-amd-ready.yml`/ + `test-amd-merge.yml` + `test-template-amd-omni.j2` + bootstrap 脚本；发布/对齐： - `release-pipeline.yaml`、`rebase-pipeline.yaml`。 -- 硬件 runner 脚本：`scripts/`（`run-amd-test.sh`、`run-xpu-test.sh`、 - `run_npu_test.sh`、nightly-index 与 wheel/镜像发布脚本）。 -- `test-ready.yml` 示例步骤组（"Simple Test"）： - `pytest tests/diffusion tests/model_executor -m 'core_model and cpu'` + - 互补 "Other Test" + "Custom Pipeline Test" - （`tests/e2e/offline_inference/custom_pipeline/ -m core_model`），跑在 +- `release/release-pipeline.yml` 是发布管线；对齐管线现在是 + `cuda/rebase-pipeline.yml`，分别上传 ready、merge 和 nightly 子管线。 +- 测试树按 owner/feature 归档：原 `tests/ar_diffusion/` 归入 + `tests/diffusion/ar_diffusion/`，full-duplex、custom pipeline、RLHF 和 ComfyUI + 归入 `tests/e2e/features//`；新增 component 测试放在对应 + `tests/{component}/`，不创建平行的旧顶层目录。 +- `cuda/test-ready.yml` 的 CPU fast lanes 仍按 `tests/diffusion`、 + `tests/model_executor`、`tests/entrypoints` 和 `tests/engine` 分组；feature lane + 使用 `tests/e2e/features/custom_pipeline/` 与 `tests/e2e/features/fullduplex/`，跑在 `gpu_1_queue`、CI docker 镜像内、`HF_HOME=/fsx/hf_cache`。 ## 运营事实（rebase-agent 观测，@122a9468） diff --git a/knowledge/repos/vllm-omni/ci/guides/test-tiers.md b/knowledge/repos/vllm-omni/ci/guides/test-tiers.md index 19476dab4..62b81cc48 100644 --- a/knowledge/repos/vllm-omni/ci/guides/test-tiers.md +++ b/knowledge/repos/vllm-omni/ci/guides/test-tiers.md @@ -1,7 +1,7 @@ --- title: "测试分级（L1–L5）与 pytest markers" created: 2026-07-16 -updated: 2026-07-31 +updated: 2026-08-05 type: guide tags: [vllm-omni, ci] sources: [docs/contributing/ci/test_system_overview.md, docs/contributing/ci/test_writing_guide.md] @@ -10,7 +10,7 @@ sources: [docs/contributing/ci/test_system_overview.md, docs/contributing/ci/tes # 测试分级（L1–L5）与 pytest markers 官方 spec：`docs/contributing/ci/test_system_overview.md` + -`test_writing_guide.md`（`main @ 807db6ef` 复核）。测试金字塔五级 + Common 规范 +`test_writing_guide.md`（`v0.26.0 @ a4ea67a2` 复核）。测试金字塔五级 + Common 规范 （PR 模板/checklist 与 CI 失败说明）。 ## 五级定义 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bb654a931..d7080e8d1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni Configuration" created: 2026-07-16 -updated: 2026-07-31 +updated: 2026-08-05 type: index tags: [vllm-omni, components, config] sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] @@ -11,10 +11,11 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] - 主要源码：`vllm_omni/config/` - 跨边界入口：`vllm_omni/diffusion/data.py`、`vllm_omni/engine/arg_utils.py`、`vllm_omni/engine/stage_init_utils.py`、`vllm_omni/engine/async_omni_engine.py` -- 主要测试：`tests/config/`、`tests/test_config_factory.py`、`tests/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 +- 主要测试：`tests/config/`、`tests/config/test_config_factory.py`、 + `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 807db6ef` 验证存在；机器基线见 +- 源码校验：以上路径在 `v0.26.0 @ a4ea67a2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/architecture.md b/knowledge/repos/vllm-omni/components/configuration/architecture.md index d72d24abd..5ac9eb022 100644 --- a/knowledge/repos/vllm-omni/components/configuration/architecture.md +++ b/knowledge/repos/vllm-omni/components/configuration/architecture.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 配置构造架构" created: 2026-07-16 -updated: 2026-07-29 +updated: 2026-08-05 type: architecture tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py] +sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/diffusion/data.py] --- # vLLM-Omni 配置构造架构 @@ -90,7 +90,7 @@ structured 与 legacy 可以有不同的最终对象，但不能有不同的字 ## PipelineConfig 与 deploy YAML 的具体结构 -以下事实曾在 `main @ 5c390096` 复核；源码会变化，动手前仍须刷新 live 版本。 +以下事实在 `v0.26.0 @ a4ea67a2` 复核；源码会变化，动手前仍须刷新 live 版本。 - **`PipelineConfig`**（模型的冻结 stage 拓扑）由模型的 `pipeline.py` 注册； **deploy YAML**（`vllm_omni/deploy/*.yaml`）只描述“这些 stage 怎么跑”。 diff --git a/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md b/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md index 373af5201..690537341 100644 --- a/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md +++ b/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni deploy YAML 实操" created: 2026-07-16 -updated: 2026-07-29 +updated: 2026-08-05 type: guide tags: [vllm-omni, components, config] sources: ["claude-workflow-starter-private@296ea45", vllm_omni/deploy/] @@ -11,7 +11,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/deploy/] 面向"要给模型写/改部署配置"的场景；schema 语义 owner 是 [Configuration](architecture.md)（本页不复制字段表）。 -`main @ 5c390096` 复核。 +`v0.26.0 @ a4ea67a2` 复核。 ## 何时需要 YAML，何时 CLI 就够 @@ -34,13 +34,16 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/deploy/] - KV 记账外分配的模型考虑 `kv_cache_memory_bytes` pin（[CONF-2a](rules.md)）。 - 争议以展开后最终配置为准（[CONF-3a](rules.md)）。 -## 代表样例（58 份 YAML 中的三类拓扑） +## 代表样例（79 份 YAML 中的三类拓扑） - 单 stage diffusion：不进 `OMNI_PIPELINES`，通常无需 YAML（引擎默认兜底），需要 固化参数时才写。 - AR+DiT 两 stage：`glm_image.yaml`、`hunyuan_image3_{ar,dit,_moe}.yaml`。 - thinker/talker(+code2wav) 多 stage：`qwen2_5_omni.yaml`（1×H100 验证）、 `qwen3_omni_moe.yaml`（2×H100 验证）、`qwen3_tts.yaml`（+ 高并发/对齐器变体）。 +- Audex 四模式各有 2B/30B YAML：`audex_{tts,tta,thinker_only,s2s}{,_30b}.yaml`； + TTS/S2S 使用 streaming codec handoff，TTA 使用同步 full-payload XCodec，30B + 必须显式选择 `_30b` 资源配置。 ## 相关 diff --git a/knowledge/repos/vllm-omni/components/configuration/omni-init-args.md b/knowledge/repos/vllm-omni/components/configuration/omni-init-args.md index e415c68f6..44946bde6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/omni-init-args.md +++ b/knowledge/repos/vllm-omni/components/configuration/omni-init-args.md @@ -31,4 +31,6 @@ sources: ["claude-workflow-starter-private@296ea45"] - 不要再依赖或推荐已不存在的 `nullify_stage_engine_defaults`。 - 增加或移动字段时，先决定它属于 orchestrator、shared 还是 per-stage engine，再修改唯一 owner 集合。 - 至少覆盖：普通 parent field 被丢弃并对非默认值告警、no-warn field 被丢弃但不告警、allowlisted field 被保留、orchestrator-only field 不泄漏、stage-local 显式值覆盖顶层 fallback。 -- 真实行为以当前 checkout 的 `tests/engine/test_arg_utils.py` 和 `tests/test_config_factory.py` 为最低回归入口；如果 example 仍引用旧 helper，只能视为待清理调用点，不能作为现行合同。 +- 真实行为以当前 checkout 的 `tests/engine/test_arg_utils.py` 和 + `tests/config/test_config_factory.py` 为最低回归入口；如果 example 仍引用旧 helper， + 只能视为待清理调用点，不能作为现行合同。 diff --git a/knowledge/repos/vllm-omni/components/configuration/pipeline-deploy-schema.md b/knowledge/repos/vllm-omni/components/configuration/pipeline-deploy-schema.md index e0b007f6e..dd5c9a49a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/pipeline-deploy-schema.md +++ b/knowledge/repos/vllm-omni/components/configuration/pipeline-deploy-schema.md @@ -1,7 +1,7 @@ --- title: "PipelineConfig 与 deploy YAML 详细 schema" created: 2026-07-16 -updated: 2026-07-29 +updated: 2026-08-05 type: guide tags: [vllm-omni, components, config] sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_config.py, docs/configuration/stage_configs.md] @@ -9,7 +9,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_conf # PipelineConfig 与 deploy YAML 详细 schema -以下事实在 `main @ 5c390096` 复核；当前稳定职责见 +以下事实在 `v0.26.0 @ a4ea67a2` 复核；当前稳定职责见 [Configuration architecture](architecture.md)，官方 spec 见 `docs/configuration/stage_configs.md` （schema 全表）与 `composable_parallel.md`。 @@ -17,7 +17,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_conf ## 双层 schema：PipelineConfig vs deploy YAML - **`PipelineConfig`**（模型的冻结 stage 拓扑）由模型的 `pipeline.py` 注册； - **deploy YAML**（`vllm_omni/deploy/*.yaml`，58 个）只描述"这些 stage 怎么跑"。 + **deploy YAML**（`vllm_omni/deploy/*.yaml`，79 个）只描述"这些 stage 怎么跑"。 未迁移模型仍走 legacy `--stage-configs-path` + `stage_args` schema （`vllm_omni/model_executor/stage_configs/*.yaml`）。 - 未显式给 `--deploy-config`/`--stage-configs-path` 时，registry 按 `model_type` @@ -50,7 +50,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_conf ## StageConfigFactory 与 pipeline registry `StageConfigFactory`（config_factory.py:47）按 `model_type` 从 -`pipeline_registry.OMNI_PIPELINES`（~44 个 key）解析出 `PipelineConfig` 或 resolver +`pipeline_registry.OMNI_PIPELINES`（51 个 key）解析出 `PipelineConfig` 或 resolver callable（如 `resolve_qwen3_omni_pipeline`）；HF `model_type` 冲突用 `hf_architectures` 消歧（如 MiMo Audio 的 HF model_type 是 qwen2）；未注册模型报错 并列出可用 key（:360）。单 stage diffusion 模型**不在**该 registry（走 @@ -66,7 +66,8 @@ callable（如 `resolve_qwen3_omni_pipeline`）；HF `model_type` 冲突用 `shutdown_unsupported_routes`（:65）——pipeline 可关闭自己不支持的 serving 路由。 - `composable_parallel/`：`--strategy-config` 把逐 stage 并行轴栈 （tp/dp/pp/ep/stage_replica 已接线；sp/cfg/vae_pp/hsdp 等保留位）以声明式 overlay - 叠加到合并后的 stage 上、先于 CLI override；**不能**与 legacy - `--stage-configs-path` 组合。 + 叠加到合并后的 stage 上、先于 CLI override；显式 deploy 值与 strategy 派生值冲突时 + fail fast，并在 worker 创建前检查 devices 与 `tp × dp × pp` world size；**不能**与 + legacy `--stage-configs-path` 组合。 源码会变化，具体函数与行号在改代码前必须以目标仓库当前版本为准。 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 5dad1d30b..a89d5e806 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 配置开发门禁" created: 2026-07-16 -updated: 2026-07-31 +updated: 2026-08-05 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/] --- # vLLM-Omni 配置开发门禁 @@ -87,6 +87,42 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "zu - 验收：每个公开 axis 都有 spec → translator → final stage config 的正向测试和 unsupported fail-fast 测试；stage 使用稳定名称。 +### CONF-4c — 新并行轴必须进入最终拓扑并由 consumer 校验 + +- 触发：新增 diffusion parallel axis、text-encoder TP、CFG/patch parallel 或把 + `strategy-config` 的声明映射到 stage runtime。 +- 强制：从 strategy spec/CLI 到 `OmniStageDiffusionParallelConfig` 和最终 process + group 展开完整链路；记录 `data × cfg × sequence × pipeline × tensor` 的设备乘积， + 并由实际 consumer 校验 rank/head divisibility、模型不支持的组合和可见设备容量。 +- 禁止：parser 接受字段后在 projection、engine kwargs 或 worker 初始化时丢失；用一个 + generic upper bound 替代模型自己的约束；用 recipe 示例或单纯 dataclass 构造代替真实 + topology/consumer 证据。 +- 验收：正向测试断言新值可从最终 stage config 读回；负向测试覆盖不整除的 head/rank、 + `cfg_parallel_size` 不适用的模型和设备不足，并在 worker 创建前失败。 + +### CONF-4d — strategy、CLI 与 pipeline-wide load balancing 必须保留唯一 owner + +- 触发：`strategy_config`、legacy YAML、`--omni-lb-policy` 或 stage replica/load + balancing 同时提供策略。 +- 强制：registry-backed strategy 才进入 translator；legacy 路径明确 warning 并保持 + 原语义；合并顺序固定为 deploy/pipeline → strategy → CLI，CLI 的显式冲突必须可见， + pipeline-wide load-balancer 只由 orchestrator 拥有并传入各 stage。 +- 禁止：接受 legacy `strategy_config` 后静默声称已生效；把 CLI 的默认值误当显式覆盖； + 每个 stage 各自构造一个相互冲突的全局 load balancer。 +- 验收：覆盖 registry、legacy、CLI 非默认值和冲突值，断言最终拓扑、设备乘积和单一 + load-balancer consumer；策略生效前若设备不足必须 fail fast。 + +### CONF-5b — `trust_remote_code` 的未指定值必须保留 deploy 优先级 + +- 触发：CLI、structured factory、legacy stage factory 或 deploy YAML 同时提供 + `trust_remote_code`。 +- 强制：调用方显式 `True`/`False` 才覆盖 per-stage deploy 值；未指定用 `None` 表示并 + 保留 YAML 值，HF config resolution 才把 `None` 收敛为安全的实际 bool。 +- 禁止：把 `store_true` 的 absent-False 当成显式 False 无条件写入 override；structured + 与 legacy 各自实现 precedence；在末端用默认值静默覆盖用户明确的 False。 +- 验收：覆盖“未指定、显式 True、显式 False”三态，并从 structured/legacy 两条路径 + 断言最终 stage config；`with_trust_remote_code_override` 是唯一合并入口。 + ### CONF-4b — 标准与 headless 启动必须解析出同一拓扑 - 触发：CLI、headless serve、offline entrypoint 或 engine factory 新增/转发配置字段。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d7e546b75..42aacd275 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,8 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 807db6ef` 验证存在 +- 源码校验：以上子模块均已在 `v0.26.0 @ a4ea67a2` 验证存在；本轮新增/扩展的 + async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 07acb6ed9..d490f3219 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-08-23 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/] confidence: high --- @@ -20,7 +20,7 @@ confidence: high | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| -| CUDA Graph、compile、fused solver、eager parity、tensor dtype/device | `execution-parity`：`DIFF-1a`–`1c` | `compile.py::regionally_compile` → runner batch → model denoise/solver | +| CUDA Graph、compile、fused solver、eager parity、tensor dtype/device、async output | `execution-parity`：`DIFF-1a`–`1d` | `compile.py::regionally_compile` → runner batch → model denoise/solver/output pump | | seed、request-local generator、guidance=0、并发 RNG、batched generators | `execution-parity`：`DIFF-1b` | `inputs/data.py::OmniDiffusionSamplingParams` → runner `_initialize_generator` → request-batch generator collate | | ModelOpt/checkpoint adapter、weight/scale remap、unknown tensor、resolution path | `checkpoint-distributed`：`DIFF-2a` | `diffusers_loader.py::{_get_checkpoint_adapter,load_weights}` → `modelopt.py::{_resolve_target_and_output_names,adapt}` | | host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | @@ -32,8 +32,8 @@ confidence: high | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| -| `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c` | -| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device | `DIFF-1a`, `DIFF-1b`, `DIFF-1c` | +| `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d` | +| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、async output readiness | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity | `DIFF-2a`, `DIFF-2b`, `DIFF-2c`, `DIFF-2d` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4f` | @@ -69,6 +69,19 @@ confidence: high - 禁止：依赖 process 默认 fp32/CPU 或隐式 cast/move；只测 shape 而不执行 BF16 consumer。 - 验收：BF16、错长、batch reorder 和 graph/eager 用例断言 consumer 收到相同 device/dtype、 alias 与结果。 ^[PR #5067] ^[PR #5068] ^[PR #5174] ^[PR #5981] +### DIFF-1d — async output 的 compute 与 output-ready 必须分成两个可观察阶段 + +- 触发：把 diffusion D2H/SHM packing 移到 worker 后台线程，或修改 + `ResultPumpThread`、`collective_rpc`、batch output split。 +- 强制：GPU compute 完成只释放下一次 forward；最终输出必须等 side-stream event、D2H + 和 SHM packing 完成后再 resolve。`result_mq` 由唯一 pump reader 分发，batch 结果按 + request 映射拆分；step-mode 保留同步路径。 +- 禁止：在 compute-done 时把未完成的 device tensor 当成最终 artifact；让多个线程同时 + 消费 result queue；用同步 `.cpu()` 掩盖 stream ordering 或让下一 step 重写源 buffer。 +- 验收：分别覆盖 compute-done、output-ready、RPC error、batch split 和 queue cleanup； + 在非 CUDA/step-mode 下证明同步 fallback 仍可构造。首批测试看 + `tests/diffusion/test_async_output_worker.py`、`test_result_pump.py` 和 + `tests/diffusion/test_ipc_async.py`。 ## Checkpoint 与分布式加载 @@ -117,6 +130,18 @@ confidence: high symlink 在同大小内容被替换后产生不同 identity；合法 Hub snapshot→blob 拓扑仍走受控快捷路径。 ^[PR #6445] +### DIFF-2e — distributed layerwise offload 只能选择一个一致的权重分片合同 + +- 触发：新增 `enable_distributed_layerwise_offload`、`dlo_use_allgather`、模型 + `OffloadPlan` 或 block discovery。 +- 强制：明确 rank 本地 CPU shard、固定双 GPU buffer、H2D/AllGather stream 和模型 + block list 的 owner；`dlo_use_allgather=false` 走每 rank full-weight 路径，开启 + AllGather 时不得再叠加已 DTensor-sharded 的 HSDP 参数。 +- 禁止：用 heuristic 找不到 block 时静默假装 offload 已启用；对 HSDP 参数二次分片； + 把 CPU 内存、AllGather 同步和设备显存开销隐藏在一个泛化的 `enable_cpu_offload` 开关。 +- 验收：CPU/Gloo 单 rank 覆盖 DTensor wrapper、shard padding、双 buffer 和 disable + cleanup；配置测试覆盖 AllGather+HSDP 的明确失败及 no-AllGather 的允许路径。 + ## 质量阈值与资源辅助 ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c72febe4e..864aac94e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -1,7 +1,7 @@ --- title: "Distributed（跨 stage 通信与数据搬运）" created: 2026-07-16 -updated: 2026-07-31 +updated: 2026-08-05 type: index tags: [vllm-omni, components, distributed] sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coordinator/, docs/design/feature/disaggregated_inference.md] @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 807db6ef` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `v0.26.0 @ a4ea67a2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） @@ -52,3 +52,4 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo | 已修过的 connector/端口产品坑 | [connector pitfalls](connector-pitfalls.md) | | 选择和配置 connector backend | [connector backends](connector-backends.md) | | 跨 stage `async_chunk` 流式语义 | [async chunk](async-chunk.md) | +| TP KV receive consensus、chunk boundary 与 active-window 合同 | [distributed rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md new file mode 100644 index 000000000..b028b8ba4 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -0,0 +1,36 @@ +--- +title: "Distributed 传输规则" +created: 2026-08-05 +updated: 2026-08-05 +type: rule +tags: [vllm-omni, components, distributed] +sources: [vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py] +confidence: high +--- + +# Distributed 传输规则 + +只有 `DIST-数字字母` 是本页可审计规则 ID。connector backend 的选择和端口分配见 +[Distributed architecture](architecture.md) 与 [connector pitfalls](connector-pitfalls.md)。 + +## DIST-1a — TP KV receive 必须做全 rank 一致的成功/放弃决定 + +- 触发：纯 TP stage 接收 KV、CFG companion 参与 KV transfer，或 connector receive + 只在部分 rank 得到 state。 +- 强制：只在 TP process group 内交换 receive state；任一 rank 发现 metadata、shape 或 + payload 不一致时，所有 rank 都走同一个 no-KV fallback，不能让部分 rank 继续 collective。 +- 禁止：用本地 `if` 丢掉 KV 后仍让其他 rank进入 receive；使用 global world group 代替 + stage 的 TP group；把 companion 当成普通 stage-0-final request 静默跳过。 +- 验收：模拟单 rank divergence，断言所有 TP rank 都放弃本次 KV 并保持后续 collective + 可继续；覆盖普通和 CFG companion 角色。 + +## DIST-1b — chunk transfer 要区分 upstream exhaustion 与本 stage 完成 + +- 触发：async-chunk/full-payload connector 传递空 segment、最后一个 chunk 或 + `WAITING_FOR_CHUNK`/active-window 状态。 +- 强制：保留空 segment 的边界语义，区分 upstream 已耗尽和当前 stage 已生成完成；收到 + 后续 chunk 时刷新 prefill/request state，并让 active-window admission 继续推进。 +- 禁止：把空 segment 当作完成信号；用本 stage 的 generation completion 终止上游；在 + connector 没有新数据时无限占用 active window 或静默丢掉 terminal update。 +- 验收：覆盖非空→空→非空、upstream exhaustion、stage completion、重复 terminal + update 和窗口恢复；首批测试看 `test_chunk_transfer_adapter.py`。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3be3c7fc3..9ef25b665 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 807db6ef` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `v0.26.0 @ a4ea67a2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/architecture.md b/knowledge/repos/vllm-omni/components/model-executor/architecture.md index 6cfac301d..aa19679cf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/architecture.md +++ b/knowledge/repos/vllm-omni/components/model-executor/architecture.md @@ -1,10 +1,10 @@ --- title: "Model Executor 共享架构" created: 2026-07-10 -updated: 2026-07-16 +updated: 2026-08-05 type: architecture tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/engine/stage_runtime.py, tests/worker/test_omni_gpu_model_runner.py] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/worker/test_omni_gpu_model_runner.py] --- # Model Executor 共享架构 @@ -23,7 +23,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/config/stage_config.py ## 当前源码职责锚点 调查前用 current main 验证这些符号仍存在；路径变化时沿调用方更新本页，不保留失效副本。 -（2026-07-16 在 `main @ 238fc0a6`（此前亦在 `dev/vllm-align @ 4f2b32c` 验证，结果一致） 复核：下列 6 个锚点文件与 +（2026-08-05 在 `v0.26.0 @ a4ea67a2` 复核：下列 6 个锚点文件与 `build_stage_runtime_overrides`、`_preprocess` 符号全部存在。） - 全局 CLI、deploy YAML 和 per-stage override 合并：`vllm_omni/config/stage_config.py` 的 `build_stage_runtime_overrides` 及其 config factory 调用方。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 9d3a82f14..8180db260 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-08-23 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, tests/worker/test_omni_gpu_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, "PR #3422", "PR #3642", "PR #4730", "PR #5074", "PR #5792", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, "PR #3422", "PR #3642", "PR #4730", "PR #5074", "PR #5792", "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -39,6 +39,44 @@ sources: [vllm_omni/worker/gpu_model_runner.py, tests/worker/test_omni_gpu_model - 禁止：不能因为严格校验开始报错，就把报错字段加入核心 dataclass、projection、known-fields 或专门白名单；不能接受字段后在分区时丢弃；不能为同一语义新增两个核心字段再补优先级；不能用“构造成功”“字段不在最终 payload”或只有测试 fixture 使用来证明字段必要。 - 验收：每个新增可接受字段都必须有 PR 前已存在或本次明确新增的真实 consumer，并有从公开入口到 consumer 的正向断言；无 consumer 或 owner 不在 stage runtime 的字段必须在入口报错；别名与 canonical 同时出现必须报冲突。最终 diff 中新增的 schema 字段数量应与 producer-consumer 表逐项一致。 +### EXEC-3b — stage loader metadata 与 stateful chunk 能力必须一同到达 consumer + +- 触发：新增 per-stage `model_subdir`/`tokenizer_subdir`、模型 architecture override， + 或模型在 async chunk 间保留执行状态。 +- 强制：从 `PipelineConfig`/deploy 合并结果把 checkpoint/tokenizer 子目录和 + `retains_state_across_chunks` 传入最终 `OmniStageModelConfig`/runner；空的 + `model_arch` 表示使用 checkpoint 自带 architectures，而不是一个空 override。 + Stateful chunk 必须与 scheduler 的容量计数和 connector requeue 合同一起审查。 +- 禁止：只在 legacy YAML 或 model helper 中记录子目录；用空字符串覆盖 checkpoint + architecture；让 scheduler 看不到仍占用 runner slot 的 parked request。 +- 验收：structured 与 legacy stage config 都能读回子目录/状态字段；Audex 等多子目录 + checkpoint 做 loader smoke；stateful async-chunk 测试证明容量上限、requeue 和 cleanup。 + +### EXEC-4a — full-duplex chunk metadata 必须按 span 隔离且可安全序列化 + +- 触发：`experimental/fullduplex` 修改 PCM/audio span、force-listen、rollback、 + resumable append、runtime-control 或 silence continuation。 +- 强制：把 speech/PCM metadata 与 rollback 状态绑定到具体 span；resumable append 重新 + arm 所需 EOS 但不得重复 turn EOS；runtime-control 输出先转换 dataclass/enum 等为 + JSON-safe 值，并在等待后重新检查 session/request 是否仍然有效。 +- 禁止：用上一 chunk 的 force-listen 或 speech marker 污染不规则下一 chunk；把 stale + session 的 silence continuation 继续发送；直接把 Python runtime object 放进控制消息。 +- 验收：覆盖不规则 PCM span、rollback、resumable append、stale continuation 和控制 + redaction；首批测试看 `tests/e2e/features/fullduplex/` 下的 input、runtime adapter + boundary 与 runtime-control redaction。 + +### EXEC-4b — shared runner 必须保留 resolved model architecture 与 model-owned hooks + +- 触发：模型 architecture override 为空、checkpoint 含多个子目录，或模型实现拥有 + native duplex/sampling policy。 +- 强制：空 `model_arch` 回退到 checkpoint architectures；loader 先按 resolved + architecture 解析 connector/checkpoint 子目录；共享 runner 只提供 typed rows 和 + hook seam，真正的 model-owned sampling/turn-boundary policy 由模型 consumer 执行。 +- 禁止：用空 override 覆盖 checkpoint metadata；只按默认架构初始化 connector；用 + generic runner policy 替换 MiniCPM/Audex 等模型自己的 native policy。 +- 验收：覆盖 blank override、多子目录 checkpoint、hook 调用顺序和 mixed-batch + request-local metadata；初始化失败必须在 scheduler/worker 继续运行前暴露。 + ## Runner 到模型的预处理合同 - 触发条件：修改或排查 runner `_preprocess` 的逐请求 metadata 生产、phase 判定、normal/batched preprocess 选择、MTP 路由条件，或多模型共用的输入预处理合同。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bb192bd1b..ce5d9b85f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -1,7 +1,7 @@ --- title: "Scheduler（AR/生成请求调度）" created: 2026-07-16 -updated: 2026-07-31 +updated: 2026-08-05 type: index tags: [vllm-omni, components, scheduler] sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache.py, docs/design/module/ar_module.md] @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 807db6ef` 验证存在：`OmniARScheduler`（:50）、 +- 源码校验：以上路径与下列类均已在 `v0.26.0 @ a4ea67a2` 验证存在：`OmniARScheduler`（:50）、 `OmniARAsyncScheduler`（:928）、`KVCacheTransferData`（:40）、`OmniGenerationScheduler`（:42）、 `OmniSchedulerMixin`（:40）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/scheduler/architecture.md b/knowledge/repos/vllm-omni/components/scheduler/architecture.md index fd84477f3..e358daeef 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/architecture.md +++ b/knowledge/repos/vllm-omni/components/scheduler/architecture.md @@ -1,15 +1,15 @@ --- title: "Scheduler 共享架构" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-08-05 type: architecture tags: [vllm-omni, components, scheduler] -sources: [docs/design/module/ar_module.md, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/prefix_cache.py, vllm_omni/worker/gpu_ar_model_runner.py] +sources: [docs/design/module/ar_module.md, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/prefix_cache.py, vllm_omni/worker/gpu_ar_model_runner.py] --- # Scheduler 共享架构 -以下事实在 `main @ 5c390096` 复核；官方叙述见 +以下事实在 `v0.26.0 @ a4ea67a2` 复核；官方叙述见 `docs/design/module/ar_module.md`（含继承 classDiagram 与请求流转 flowchart）。 ## 继承链（对 vLLM 的扩展方式） @@ -53,6 +53,13 @@ token，前缀行从缓存重建后拼入跨 stage payload。消费端合同： `higgs_audio_v3_talker.py:236` 显式声明 False）。该机制的失败模式与硬规则见 [rules](rules.md) 的 `SCHED-1a`。 +在目标版本，`OmniGenerationScheduler` 还区分 +`retains_state_across_chunks`：等待 connector chunk 的 request 仍占用 model-runner +capacity，并在 full-payload chunk 到达后重新进入可调度队列。AR scheduler 在消费 +sampled-token logprobs 前完成 request-local 合同校验；stage-0-final 的普通请求可以 +跳过下游 KV，但 companion 等显式带 `omni_force_kv_transfer` 的请求不能走该 shortcut。 +这些跨层语义分别见 [scheduler rules](rules.md) 的 `SCHED-5a`–`SCHED-5c`。 + ## 与 orchestrator 的边界 调度器只负责单 stage 内的请求生命周期与跨 stage 载荷的调度面；逻辑请求在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 9233a817c..81d5f3bae 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -1,10 +1,10 @@ --- title: "Scheduler 规则" created: 2026-07-16 -updated: 2026-07-31 +updated: 2026-08-05 type: rule tags: [vllm-omni, components, scheduler] -sources: ["vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, "PR #4106"] +sources: ["vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, "PR #4106"] --- # Scheduler 规则 @@ -24,6 +24,8 @@ PR 描述先命中下表，再打开对应规则组和首批源码；changed fil | `max_num_batched_tokens`、prefill throttle、低预算 GPU hang | SCHED-2a | `core/sched/omni_ar_scheduler.py::OmniARScheduler.schedule`；`core/sched/omni_generation_scheduler.py::OmniGenerationScheduler.schedule`；触发它的 deploy/test 配置 | | vLLM bump、scheduler rebase、KV connector stats | SCHED-3a | 两个 scheduler 的 `schedule` / `update_from_output` 与 live upstream `vllm/v1/core/sched/` | | side-stream D2H、pinned host tensor、源 buffer 复用 | SCHED-4a/4b | `worker/gpu_ar_model_runner.py::_copy_tensor_payload_to_cpu`、`_get_or_create_omni_payload_copy_stream`；`core/prefix_cache.py` 的 async copy 路径 | +| sampled-token logprobs、spec decode trim、request-local output error | SCHED-5a | `core/sched/omni_ar_scheduler.py::_slice_sampled_logprobs`、`update_from_output` | +| stateful async chunk、full-payload input、KV cleanup | SCHED-5b/5c | `core/sched/omni_generation_scheduler.py`、`omni_scheduling_coordinator.py`、`omni_ar_scheduler.py::_free_request` | 若描述只写模型症状，先从模型 owner 找到 payload producer/consumer；只有实际断点落在调度、 prefix cache 或 copy lifetime 时才把 Scheduler 加为 owner。 @@ -146,6 +148,52 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 验收：CPU-only 环境能构造并走同步 fallback；CUDA 环境仍使用 pinned + async，两个 分支产生相同内容。 ^[PR #4106] +## SCHED-5a — sampled-token logprobs 先校验再修改 request + +- 触发：AR scheduler 消费 model-runner sampled tokens 和 `num_logprobs`，尤其是 + spec-decode 或 batched output slicing。 +- 强制：按 request 切出 logprob rows，校验二维 shape、行数、第一 token 对齐和有限值； + 失败只将当前 request 置为 `FINISHED_ERROR`，不能先 append token 或污染同批 request。 + stop/EOS 截断后再按最终 emitted token 数重新 slice。 +- 禁止：`logprobs` 缺失时静默跳过；用 truthiness 判断合法空输出；把 runner 的错位 + row 当作用户请求成功。 +- 验收：覆盖 missing、wrong shape/count、token misalignment、NaN、spec-decode trim + 和同批健康 request 继续完成。 + +## SCHED-5b — stateful async-chunk request 必须占用调度容量 + +- 触发：模型在等待下一 chunk 时保留 runner state，或 full-payload connector 负责把下 + 一段重新送入 generation stage。 +- 强制：`retains_state_across_chunks` 为真时，把 connector 中等待 chunk 的 request + 计入 `max_num_seqs`；full-payload consumer 在 chunk 到达时重新进入 waiting queue， + 不要被 base scheduler 提前停放。 +- 禁止：只按 `running` list 计数导致超额 admission；把 connector-fed chunk 当成 API + streaming update；仅在 abort 路径释放 receiver。 +- 验收：mixed batch 覆盖等待 chunk 的容量上限、full-payload requeue、normal finish + 的 receiver cleanup 和 abort/replica-loss cleanup。 + +## SCHED-5c — stage-0 final request 的 KV transfer 例外必须显式标记 + +- 触发：stage-0 的 text/final 输出通常不需要下游 KV，但 CFG companion 或其他终端 + payload 仍需要复用该 cache。 +- 强制：用 `omni_force_kv_transfer` 等 request metadata 明确覆盖“final stage=0”的 + shortcut；scheduler、engine 和 companion tracker 使用同一个标记语义。 +- 禁止：按 final stage id 静默跳过所有 stage-0 KV；只在 companion 的 producer 设置 + 标记而不测试 scheduler 的 transfer decision。 +- 验收：普通 stage-0-final request 不传 KV，标记 request 传 KV，且 metadata 在 + `OmniARScheduler._request_omits_kv_transfer_to_next_stage` 中可读回。 + +## SCHED-5d — CFG companion 必须成对推进，缺失时有限降级并终止等待 + +- 触发：Audex/扩散 CFG companion、双 waiting queue、不同 chunk 进度或 companion + abort/split。 +- 强制：不完整 pair 暂存并按 parent/companion 对齐进度；完成时一起收敛，缺失或拆分 + 的 companion 只能走显式、有界的 fallback，并清理两侧状态。 +- 禁止：只推进 parent 让 companion 永久停在 waiting；把缺 companion 当普通 batch + 请求静默放行；用无限等待掩盖 replica loss 或 abort。 +- 验收：覆盖完整 pair、不同进度、missing/split、parent abort 和 companion abort， + 断言请求不会挂死、错误归属保持 request-local、队列和 connector state 都释放。 + ## 相关 - 机制与边界见 [architecture](architecture.md)；跨 stage 数据面见 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1cdeb8f6e..3f62f7d44 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 807db6ef` 验证存在 +- 源码校验：以上路径均已在 `v0.26.0 @ a4ea67a2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index b5a2056c3..9faeb0310 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-08-23 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5157", "PR #5670", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/metrics/prometheus.py] +sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5157", "PR #5670", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py] confidence: high --- @@ -37,6 +37,7 @@ confidence: high | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c` + 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d`, `SERV-5d` | | `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵 | `SERV-5a`, `SERV-5b`, `SERV-5c` | +| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6c` | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`, `SERV-4b`, `SERV-4c`, `SERV-4d`, `SERV-4e`, `SERV-4f`, `SERV-4g`, `SERV-4h` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | @@ -225,5 +226,42 @@ confidence: high - 验收：把 handler 置空、移除 route/method、绕过 inner app 三类 mutation 均使 guard 失败； 正常 app 的 HEAD/OPTIONS census 与必需 state key 完整。 ^[PR #6202] +## Full-duplex 与 CFG companion 生命周期 + +### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology + +- 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 +- 强制：stage-0 的首次 `submit_initial` 在 async-chunk 开启时预热后续 stage 的 + runtime/pool；每个 stage 保留自己的 fence、submit timestamp 和 request state。 +- 禁止：等到 duplex audio/control 事件真正抵达才首次启动下游 stage；用一个全局 fence + 表示多 stage readiness；prewarm 失败后仍发布 generation-ready。 +- 验收：覆盖首次 stage-0 submit、后续 stage submit、重复 update 和 prewarm failure； + CPU/mock 测试必须证明 request state 与 stage pool 生命周期一致。 + +### SERV-6b — CFG companion 输出要非破坏性聚合并在所有 teardown 路径清理 + +- 触发：CFG companion、deferred parent、streaming terminal update、abort、stage error + 或 replica loss。 +- 强制：parent re-submit 前 companion outputs 可重复读取；parent 只在全部 companion + 完成后释放；companion error/abort 必须给 deferred parent 发送结构化错误，并由统一 + cleanup 路径清理 parent、companion、tracker 和 stage-pool binding。 +- 禁止：用 destructive `pop` 让第二次 terminal update 丢 companion；只清 parent 不清 + companion；companion 永久等待时静默挂起 parent。 +- 验收：覆盖正常全量完成、重复读取、companion error、parent abort、companion abort + 和 replica loss，断言没有遗留 tracker state 或未完成的 deferred parent。 + +### SERV-6c — duplex capability gate 必须区分未启用与配置失败 + +- 触发：`session_mode=duplex`、`/v1/realtime?duplex=true`、runtime extension 或 + duplex deploy configuration 发生变化。 +- 强制：入口只在明确的 duplex opt-in 下建立 handler；能力检查必须保留结构化配置 + 错误与“不支持该模式”的区别，session、request、stage fence、lease 和 ordered + mailbox 的生命周期必须在 disconnect、abort、expiry 与 replica loss 时一起终止。 +- 禁止：把所有 config-load exception 折叠成 `duplex unavailable`；用普通 streaming + handler 冒充持久 session；用全局 fence 或无界 pending input 替代 per-session bound。 +- 验收：覆盖未 opt-in、有效 opt-in、malformed config、stale fence、lease expiry、 + disconnect 和 terminal response ordering；控制消息中的 dataclass/enum/tensor 必须 + 在发送前变成 JSON-safe 值。 + 请求到 engine 的边界见 [Serving architecture](architecture.md)；公开协议通用检查见 [review contracts](../../../../general/review/guides/reviewer-lens-contracts.md)。 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index 20ae4efdb..f02a2fd4f 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -1,7 +1,7 @@ --- title: "官方设计文档地图（docs/design/**）" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-08-05 type: guide tags: [vllm-omni, docs] sources: [docs/design/index.md, docs/design/architecture_overview.md] @@ -9,7 +9,7 @@ sources: [docs/design/index.md, docs/design/architecture_overview.md] # 官方设计文档地图（docs/design/**） -`main @ 5c390096` 复核。注意：官方 `docs/design/index.md` 只列了子集（Architecture +`v0.26.0 @ a4ea67a2` 复核。注意：官方 `docs/design/index.md` 只列了子集（Architecture Overview、5 篇 feature、metrics、3 篇 module）；**完整树比索引大得多**—— `feature/` 下还有 async_chunk、cache_dit、teacache、prefix_caching、7 篇并行策略、 `omni_connectors/` 逐后端 spec 等未入索引的文档，找 spec 时直接 `ls docs/design/feature/`。 diff --git a/knowledge/repos/vllm-omni/models/audex/_index.md b/knowledge/repos/vllm-omni/models/audex/_index.md new file mode 100644 index 000000000..99c754650 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/audex/_index.md @@ -0,0 +1,51 @@ +--- +title: "Nemotron-Labs Audex" +created: 2026-08-05 +updated: 2026-08-05 +type: index +tags: [vllm-omni, models, serving] +sources: [vllm_omni/model_executor/models/audex/, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/audex_tts.yaml, vllm_omni/deploy/audex_tts_30b.yaml, recipes/NVIDIA/Nemotron-Labs-Audex.md, examples/offline_inference/audex/, examples/online_serving/audex/] +confidence: high +--- + +# Nemotron-Labs Audex + +## 名称、源码与注册 + +- checkpoints：`nvidia/Nemotron-Labs-Audex-2B` 与 + `nvidia/Nemotron-Labs-Audex-30B-A3B`；共享 `audex` model-executor 目录，包含 + thinker、codec、decoder、pipeline 和 stage input processor。 +- AR registry 新增五个 architecture：`NemotronDenseForCausalLM`、 + `AudexCode2Wav`、`AudexXCodec1`、`NemotronDenseAudexForConditionalGeneration`、 + `NemotronHAudexForConditionalGeneration`。 +- pipeline registry 提供 `audex_tts`、`audex_tta`、`audex_thinker_only`、 + `audex_s2s`；repo-root 的 `model_type: nemotron_labs_audex` 是默认 TTS + pipeline 的显式 alias，不应误路由到 thinker-only。 + +## 四种 pipeline 的 stage 合同 + +- TTS：thinker 逐 chunk 产生 codec frames，经 streaming `AudexCode2Wav` 解码； + terminal chunk 带 `stream_finished`，不能把空 codec token 当作 engine failure。 +- S2S：text 走 stage 0 的最终文本输出，audio payload 继续送到 `Code2Wav`；两类输出 + 必须按同一 request 对齐，不能把 audio bridge 当成普通文本 completion。 +- thinker-only：单 stage audio understanding，不自动追加 speech decoder；TTA 则是 + synchronous full-payload 路径，使用完整 RVQ payload 交给外部 `AudexXCodec1` 解码。 +- 这些 stage/shape 语义由 `stage_input_processors/audex.py` 和各模式 deploy 共同定义； + 只通过 registry import 或单一 TTS smoke 不能证明其他三种模式的 handoff。 + +## 部署边界 + +- 四种模式各有 2B/30B deploy YAML。TTS、TTA、thinker-only 和 S2S 的入口与输入/输出 + 组合以官方 recipe 为准；共享 16 kHz 流式 decoder 不意味着所有模式都提供 speech output。 +- 30B-A3B 必须显式选择对应的 `*_30b.yaml`；不能让 repo-root 自动检测复用 2B 的默认 + stage topology。改 deploy 时同时核对 stage 显存预算、Mamba prefix-cache 约束和 + decoder 的跨 stage payload。 +- Audex 的模型专属事实留在本页和 recipe；共享 stage/config/serving 合同分别归 + [Model Executor](../../components/model-executor/_index.md)、[Configuration](../../components/configuration/_index.md) + 和 [Serving](../../components/serving/_index.md)。 + +## 验证入口 + +离线脚本位于 `examples/offline_inference/audex/`，在线客户端和 server launcher 位于 +`examples/online_serving/audex/`。审查新模式时至少按目标 mode 绑定对应 deploy、endpoint +和音频输出类型；单独通过 registry import 不能证明 stage handoff 或服务合同。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 05bb0d10e..c84dbd8cb 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -1,7 +1,7 @@ --- title: "模型代码入口与 registry 快照" created: 2026-07-16 -updated: 2026-07-31 +updated: 2026-08-05 type: guide tags: [vllm-omni, models] sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/] @@ -9,8 +9,8 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 -本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数仍是 -`v0.26.0rc1 @ 807db6ef`（2026-07-28）快照，数字会漂移，不能凭它断言“不支持”。 +本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 +`v0.26.0 @ a4ea67a2`（2026-08-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -35,36 +35,38 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| -| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 72 个架构名 / 26 个模型族目录 | -| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 61 条 pipeline / 37 个模型族目录 | -| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 46 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 71 份 | +| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 77 个架构名 / 27 个模型族目录 | +| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 58 条 pipeline / 38 个模型族目录 | +| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 51 个 key | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 79 份 | -对比上一审计快照（`5d44868e`,2026-07-21）：AR 架构 69→72，diffusion -pipeline 59→61，OMNI_PIPELINES 保持 46，deploy 65→71。新增 diffusion -家族是 `boogu_image` 和 `lingbot_video`；AR 新增项属于已有的 -`mammoth_moda2` 与 `minicpmo_4_5` 家族。 +对比上一审计快照（`807db6ef`）：AR 架构 72→77，diffusion pipeline 61→58， +OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusion 新增 +`minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 +pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 +Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -## AR/omni 模型族（26） +## AR/omni 模型族（27） -aura_omni、bagel、cosyvoice3、covo_audio、dynin_omni、fish_speech、glm_image、 +aura_omni、audex、bagel、cosyvoice3、covo_audio、dynin_omni、fish_speech、glm_image、 glm_tts、higgs_audio_v2、higgs_audio_v3、hunyuan_image3、indextts2、 mammoth_moda2、mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、moss_tts、 moss_tts_nano、omnivoice、qwen2_5_omni、qwen3_omni、qwen3_tts、step_audio2、 voxcpm2、voxtral_tts -## Diffusion 模型族（37） +## Diffusion 模型族（38） audiox、bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamid_omni、 dreamzero、ernie_image、flux、flux2、flux2_klein、glm_image、gr00t、helios、 hidream_image、hunyuan_image3、hunyuan_video、internvla_a1、krea2、lance、 lingbot_video、longcat_image、ltx2、magi_human、ming_flash_omni、nextstep_1_1、omnigen2、 omnivoice、ovis_image、qwen_image、sd3、sdxl、sensenova_u1、soulx_singer、 -stable_audio、wan2_2、z_image +stable_audio、wan2_2、z_image、minimax_h3 -## OMNI_PIPELINES key（46） +## OMNI_PIPELINES key（51） -Gr00tN1d7（注意:唯一 CamelCase key）、aura_omni、bagel、bagel_single_stage、 +Gr00tN1d7（注意:唯一 CamelCase key）、aura_omni、audex_s2s、audex_thinker_only、 +audex_tta、audex_tts、bagel、bagel_single_stage、 bagel_think、cosyvoice3、covo_audio、dreamzero、dynin_omni、fish_qwen3_omni、 glm_image、glm_tts、higgs_audio_v2、higgs_multimodal_qwen3、hunyuan_image3_ar、 hunyuan_image3_dit、hunyuan_image_3_moe、hunyuan_video_15、indextts2、lance、 @@ -74,7 +76,7 @@ ming_tts、ming_tts_moe、minicpmo_4_5、moss_tts_delay、moss_tts_local、 moss_tts_nano、moss_tts_realtime、omnivoice、qwen2_5_omni、 qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_tts、 soulxsinger_svc、soulxsinger_svs、step_audio_2、step_audio_2_asr、voxcpm2、 -voxtral_tts、wan2_2_ti2v +voxtral_tts、wan2_2_ti2v、nemotron_labs_audex 注意：单 stage diffusion 模型**多数不在** `OMNI_PIPELINES`（引擎为它们生成 默认 diffusion stage 配置,见 [Config 组件](../components/configuration/architecture.md)）; @@ -85,8 +87,8 @@ voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 5d44868e \ - --to v0.26.0rc1 \ + --from 807db6ef \ + --to a4ea67a2 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/dreamzero/architecture.md b/knowledge/repos/vllm-omni/models/dreamzero/architecture.md index 738b4b99d..0f0763f77 100644 --- a/knowledge/repos/vllm-omni/models/dreamzero/architecture.md +++ b/knowledge/repos/vllm-omni/models/dreamzero/architecture.md @@ -66,14 +66,14 @@ sources: [vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py, vllm_omni/ ## 怎样验证功能、精度和性能 -pin 上有**上游一致性测试**（`tests/dreamzero/upstream/` 对上游 socket -server 的 e2e 源一致性）与较全单测;本次调查未发现性能 gate——但有 +pin 上有 `tests/e2e/accuracy/test_dreamzero.py` 与较全的 +`tests/diffusion/models/dreamzero/` 单测;本次调查未发现性能 gate——但有 `DZ_PHASE_TIMING=1` 分相计时插桩（逐 mark 同步 CUDA,**benchmark 时应 关闭**,源注释明示）。共享设施见 [Diffusion 组件](../../components/diffusion/_index.md)。 -- 单测：`tests/dreamzero/`（crossattn cache、fused_qk_rms_norm、QKV 融合、 - pipeline state、utils、OpenPI helper）;e2e +- 单测：`tests/diffusion/models/dreamzero/`（crossattn cache、fused_qk_rms_norm、 + QKV 融合、pipeline state、utils、OpenPI helper）;e2e `tests/e2e/online_serving/test_dreamzero_expansion.py`;配置解析 `tests/entrypoints/test_resolve_dreamzero_config.py`;示例 `examples/{offline_inference,online_serving}/dreamzero/` diff --git a/knowledge/repos/vllm-omni/models/gr00t/architecture.md b/knowledge/repos/vllm-omni/models/gr00t/architecture.md index 6edd8cd81..165e677b3 100644 --- a/knowledge/repos/vllm-omni/models/gr00t/architecture.md +++ b/knowledge/repos/vllm-omni/models/gr00t/architecture.md @@ -61,7 +61,7 @@ sources: [vllm_omni/diffusion/models/gr00t/pipeline_gr00t.py, vllm_omni/diffusio pin 上有**位级回归**（对 Isaac-GR00T ZMQ 参考值 max_diff=0.0)——这是行为 正确性 gate;无性能 gate;examples 缺失。 -- e2e：`tests/e2e/online_serving/test_gr00t_openpi.py` +- e2e：`tests/e2e/online_serving/test_gr00t_openpi_expansion.py` （`GR00T_NOISE_SEED=42`,init 1200 s/stage 900 s,缺 `websockets`+ `openpi_client` 即跳过）;测试客户端 `tests/gr00t/openpi_client_helper.py`; 单测 `tests/diffusion/models/gr00t/test_pipeline.py`（stub policy,观测 diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md b/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md index 341ef44e2..5035cb7d8 100644 --- a/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md +++ b/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md @@ -61,7 +61,7 @@ sources: [vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.p ## 怎样验证功能、精度和性能 -- 单元：`tests/unit/higgs_audio_v3/test_higgs_audio_v3.py`（无 GPU,AC-1..10: +- 单元：`tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py`（无 GPU,AC-1..10: config/prompt/融合模块/delay/stage processor/registry）。 - e2e：v2 `tests/e2e/{offline_inference,online_serving}/test_higgs_audio_v2_expansion.py`; v3 `tests/e2e/online_serving/test_higgs_audio_v3.py`;perf diff --git a/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md b/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md index c57f88715..b9b7416a6 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md @@ -29,7 +29,7 @@ diffusion 视频,AR registry 无入口;image3 的结构见该页）。 **I2V 不在 OMNI_PIPELINES**（走单 stage diffusion 兜底）。pipeline config **无 `default_deploy_config_name`**——`hunyuan_video_15.yaml` 不会 自动加载;显式传裸文件名时按 `_DEPLOY_DIR` 解析（pin 上仅 - `tests/test_config_factory.py` 按名加载它）。 + `tests/config/test_config_factory.py` 按名加载它）。 - T2V vs I2V 差异速览：I2V 加 SigLIP 图像编码器 + 图像预处理（从 `max_area` 推 H/W）+ 首帧 VAE 条件 latent/掩码;T2V 供零 image_embeds、 零 cond_latents、零 mask。 diff --git a/knowledge/repos/vllm-omni/models/ltx2/_index.md b/knowledge/repos/vllm-omni/models/ltx2/_index.md index 65eb7273f..36b15c09d 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/_index.md +++ b/knowledge/repos/vllm-omni/models/ltx2/_index.md @@ -1,10 +1,10 @@ --- title: "LTX-2 家族（含 LTX-2.3）" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-08-05 type: index tags: [vllm-omni, models, ltx2] -sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, recipes/LTX/LTX-2.3.md] +sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, recipes/LTX/LTX-2.md] --- # LTX-2 家族（含 LTX-2.3） @@ -14,12 +14,12 @@ sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, rec - 厂商/模型：Lightricks；22B 参数文本→视频+音频生成（T2V/I2V，48kHz 同步音频， 768x512 可达 20+ 秒）；Diffusers 格式 checkpoint `dg845/LTX-2.3-Diffusers` - 源码：`vllm_omni/diffusion/models/ltx2/`（纯 diffusion，无 AR stage） -- registry（`vllm_omni/diffusion/registry.py`，`main @ 5c390096` 验证）： - LTX-2 条目 `LTX2Pipeline`（:69）/`LTX2ImageToVideoPipeline`（:74）/ - `LTX2TwoStagesPipeline`（:79）+ DMD2 蒸馏变体；LTX-2.3 条目 `LTX23Pipeline` - （:99）/`LTX23ImageToVideoPipeline`（:104），后处理共用 - `get_ltx2_post_process_func`（:504） -- 官方 recipe：`recipes/LTX/LTX-2.md`、`recipes/LTX/LTX-2.3.md` +- registry：`LTX2Pipeline` 统一 LTX-2/LTX-2.3 的 one-stage T2V/I2V； + `LTX2DistilledPipeline` 统一 distilled two-stage T2V/I2V；DMD2 仍由 + `LTX2T2VDMD2Pipeline`/`LTX2I2VDMD2Pipeline` 提供。旧的 LTX23、ImageToVideo 和 + TwoStages registry names 已删除，没有兼容 alias。 +- 官方 recipe 已合并为 `recipes/LTX/LTX-2.md`；已删除的 + `recipes/LTX/LTX-2.3.md` 不能继续作为 source 或事实依据。 - 依赖共享 [Diffusion 组件](../../components/diffusion/_index.md) ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/ltx2/architecture.md b/knowledge/repos/vllm-omni/models/ltx2/architecture.md index 0d957e1bd..b2464d49a 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/architecture.md +++ b/knowledge/repos/vllm-omni/models/ltx2/architecture.md @@ -1,15 +1,15 @@ --- title: "LTX-2/2.3 模型架构与证据索引" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-08-05 type: architecture tags: [vllm-omni, models, ltx2] -sources: [recipes/LTX/LTX-2.3.md, vllm_omni/diffusion/registry.py, "#4381", "#4464"] +sources: [recipes/LTX/LTX-2.md, vllm_omni/diffusion/registry.py, "#4381", "#4464"] --- # LTX-2/2.3 模型架构与证据索引 -以下事实在 `main @ 5c390096` 复核（recipe 与 registry）。 +以下事实在 `v0.26.0 @ a4ea67a2` 复核（合并后的 recipe 与 live registry）。 ## 结构与 serving @@ -17,12 +17,15 @@ sources: [recipes/LTX/LTX-2.3.md, vllm_omni/diffusion/registry.py, "#4381", "#44 T2V 与 I2V 皆可，输出带 48kHz 同步音频。验证建议从 96GB 级 GPU 起步 （recipe 原文）。 - serving 入口（LTX-2.3）： - `vllm serve dg845/LTX-2.3-Diffusers --omni --model-class-name LTX23Pipeline - --stage-init-timeout 600`；需要 `diffusers >= 0.38.0`（git 安装）。 -- pipeline 变体：常规单段（`LTX2Pipeline`/`LTX23Pipeline`）、I2V - （`*ImageToVideoPipeline`）、两段式（`LTX2TwoStagesPipeline`）与 DMD2 蒸馏 - （`LTX2T2VDMD2Pipeline`/`LTX2I2VDMD2Pipeline`）；LTX-2.3 后处理复用 ltx2 的 - `get_ltx2_post_process_func`。 + `vllm serve diffusers/LTX-2.3-Diffusers --omni --stage-init-timeout 600`；统一 + `LTX2Pipeline` 由 checkpoint metadata 选择 LTX-2 或 LTX-2.3 profile。 +- pipeline 变体：one-stage 使用 `LTX2Pipeline`，distilled two-stage 使用 + `LTX2DistilledPipeline`，DMD2 使用 `LTX2T2VDMD2Pipeline`/ + `LTX2I2VDMD2Pipeline`；T2V/I2V 通过是否提供 `image=` 选择，不再使用单独的 + `*ImageToVideoPipeline` registry names。 +- Python `forward` 只允许 `req` 作为 positional argument，其他参数必须 keyword-only； + 这对直接调用者和显式 `--model-class-name` 覆盖是 breaking change，CLI/HTTP recipe + 已按 named fields 调用。 - 单段 diffusion 模型不在 `OMNI_PIPELINES` registry（走 `async_omni_engine.py` 的默认 diffusion stage 兜底），deploy 语义见 [Config 组件](../../components/configuration/architecture.md)。 diff --git a/knowledge/repos/vllm-omni/models/mammoth-moda2/architecture.md b/knowledge/repos/vllm-omni/models/mammoth-moda2/architecture.md index 9185381b6..5de84473a 100644 --- a/knowledge/repos/vllm-omni/models/mammoth-moda2/architecture.md +++ b/knowledge/repos/vllm-omni/models/mammoth-moda2/architecture.md @@ -75,7 +75,7 @@ pin 上只有**功能/config 面**的验证入口,没有专门的精度基线或 - e2e：`tests/e2e/offline_inference/test_mammoth_moda2_expansion.py` （t2i + AR）;config 单测 - `tests/unit/mammoth_moda2/test_mammoth_moda2_config.py`;AR 图像理解示例 + `tests/model_executor/models/mammoth_moda2/test_mammoth_moda2_config.py`;AR 图像理解示例 `examples/offline_inference/mammothmodal2_preview/`。 - 已知未决：runner 侧把 hidden 打进 `multimodal_output["latent"]` 的确切 代码位、`Mammoth2DecoderLayer` 每层 moe 布线、DiT 去噪逐步数学——分析时 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md new file mode 100644 index 000000000..7bdf663e3 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -0,0 +1,40 @@ +--- +title: "MiniMax H3" +created: 2026-08-05 +updated: 2026-08-05 +type: index +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, vllm_omni/entrypoints/openai/video_api_utils.py] +confidence: high +--- + +# MiniMax H3 + +## 名称、源码与任务 + +- checkpoint：`MiniMaxAI/MiniMax-H3`；纯 diffusion registry architecture 是 + `MiniMaxH3Pipeline`，实现位于 `diffusion/models/minimax_h3/`。 +- 支持 `t2va`、`fl2va`、`ref2va` 三种 joint video+audio 条件模式；一个 server 进程 + 一次只加载 `FL2VA` 或 `Ref2VA` 分区，不能把两个分区当成同一份权重同时服务。 +- 输出是带同步音频的 MP4；参考视频、图像和音频的预处理合同由 pipeline 与 video API + 共同决定，不能仅凭 endpoint 名称推断输入组合。 +- 生成合同固定为 24 FPS 视频与 32 kHz 音频；空间尺寸必须按 32 对齐，宽高比限制在 + `1:4` 到 `4:1`，这些约束在 request validation 阶段执行而不是由 VAE 静默修正。 + +## 并行与加载约束 + +- H3 是 CFG-distilled pipeline，`cfg_parallel_size` 必须保持 `1`。VAE patch parallel + 使用 H3 原生 `tile` 模式；`text_encoder_tp_size` 作用在 DiT group 的前 N 个 rank， + 并且必须同时整除 Qwen3-VL 的 64 个 attention heads 与 8 个 KV heads。 +- 单卡 accuracy 路径使用 CPU offload；多卡部署可使用 Ulysses、text-encoder TP、VAE + tile/patch parallel 或 layerwise offload，但每个组合都必须按最终并行拓扑和设备数验证。 +- 音频加载优先使用 torchaudio；当 TorchCodec/torchaudio 在 CPU-only aarch64 环境不可用 + 时，`reference_video.load_audio_file` 回退到 soundfile，再对 libsndfile 不支持的格式 + 通过 ffmpeg 转 WAV。该回退保持 `(channels, samples)` float32 与原始 sample rate 合同。 + +## 验证入口 + +模型专属 contract、packing、parallel 和 e2e 测试在 `tests/diffusion/models/minimax_h3/`。 +硬件 recipe 只记录已验证的 GPU/NPU 形状；性能数字不能从 recipe 的配置示例泛化为全硬件 +保证。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 +[Configuration](../../components/configuration/_index.md) 和 [Serving](../../components/serving/_index.md)。 diff --git a/knowledge/repos/vllm-omni/models/soulx-singer/_index.md b/knowledge/repos/vllm-omni/models/soulx-singer/_index.md index e8986fd65..b21d9a267 100644 --- a/knowledge/repos/vllm-omni/models/soulx-singer/_index.md +++ b/knowledge/repos/vllm-omni/models/soulx-singer/_index.md @@ -59,7 +59,7 @@ sources: [vllm_omni/diffusion/models/soulx_singer/, vllm_omni/model_executor/mod - extra-body 面在 pipeline 类的 ClassVar（无 model_extras 文件）：SVS 收 metadata/语言/control/max_merge_duration…出 `f0_shift`;SVC 收 wav/F0 路径 出 `pitch_shift`（`auto_shift` 支持,SVC 移调 = `pitch_shift×5` 粗 F0 bin）。 -- 自动化测试仅见离线 e2e（`test_soulxsinger.py`）;本次调查未发现在线示例 +- 自动化测试仅见离线 e2e（`test_soulxsinger_expansion.py`）;本次调查未发现在线示例 的 CI 覆盖。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/soulx-singer/architecture.md b/knowledge/repos/vllm-omni/models/soulx-singer/architecture.md index 5a79f7ec7..8f46d6656 100644 --- a/knowledge/repos/vllm-omni/models/soulx-singer/architecture.md +++ b/knowledge/repos/vllm-omni/models/soulx-singer/architecture.md @@ -59,7 +59,7 @@ sources: [vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_base.py, 自动化测试仅覆盖离线 e2e,另有一个可复跑基准;本次调查未发现在线链路的 CI 覆盖——精度/性能结论需另行实测。 -- e2e：`tests/e2e/offline_inference/test_soulxsinger.py`（快照双仓、建 +- e2e：`tests/e2e/offline_inference/test_soulxsinger_expansion.py`（快照双仓、建 SVS/SVC view 目录、校验 phone_set.json;资产 `tests/assets/soulxsinger/`）;示例 `examples/offline_inference/text_to_speech/soulxsinger/end2end.py` 与 diff --git a/knowledge/repos/vllm-omni/rebase/_index.md b/knowledge/repos/vllm-omni/rebase/_index.md index 6cfef22a4..16e1bc866 100644 --- a/knowledge/repos/vllm-omni/rebase/_index.md +++ b/knowledge/repos/vllm-omni/rebase/_index.md @@ -1,16 +1,16 @@ --- title: "Rebase（对齐 upstream vLLM）" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-08-05 type: index tags: [vllm-omni, rebase] -sources: [".buildkite/rebase-pipeline.yaml", "vllm-omni-rebase-agent@122a9468:agent/config.py"] +sources: [".buildkite/cuda/rebase-pipeline.yml", "vllm-omni-rebase-agent@122a9468:agent/config.py"] --- # Rebase（对齐 upstream vLLM） 把 vllm-omni 对齐到新版 upstream vLLM 的周期性工作：专用分支 `dev/vllm-align`、 -专用管线 `.buildkite/rebase-pipeline.yaml`（`main @ 5c390096` 验证存在）、专职 +专用管线 `.buildkite/cuda/rebase-pipeline.yml`（`v0.26.0 @ a4ea67a2` 验证存在）、专职 自动化（rebase-agent 仓库——运营系统在那里，本主题只沉淀领域知识）。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/rebase/upstream-api-drift.md b/knowledge/repos/vllm-omni/rebase/upstream-api-drift.md index eb8b0da92..0194c5290 100644 --- a/knowledge/repos/vllm-omni/rebase/upstream-api-drift.md +++ b/knowledge/repos/vllm-omni/rebase/upstream-api-drift.md @@ -33,7 +33,7 @@ modules=[scheduler]，status=active，run_count=20，2026-06-15 创建 / 07-11 `use_harmony: bool = False` 作类属性；被删方法在 `OmniOpenAIServingChat` 恢复为本地副本；核对 `vllm.entrypoints.openai.parser.harmony_utils` 的 import ——`parse_chat_output` 若从该模块移除则内联。回归：`tests/entrypoints/test_stream_finish_reason.py` - + `tests/comfyui/test_comfyui_integration.py::test_understanding_node`； + + `tests/e2e/features/comfyui/test_comfyui_integration.py::test_understanding_node`； pre-commit（ruff format 可能内联被删 import）。 ^[SK-fix-omni-serving-chat-upstream-harmony-refactor] - 细化实例（debug-memory #184，key=`missing_should_check_unstreamed_tool_arg_tokens`， @@ -192,8 +192,9 @@ modules=[online_serving, model_executor]，status=active，run_count=6， - 验证：`cd /rebase/vllm-omni && /rebase/.venv/bin/python -m pytest --collect-only -q ` → rc=0 且 `N tests collected`；重跑 pipeline 测试进入执行。 -- 禁止：为满足陈旧配置**重造上游已删除的测试文件**（曾手写 - `tests/e2e/online_serving/test_hunyuan_video_15.py`——恢复上游有意删除的覆盖、与幸存者重复、以后每次 +- 禁止：为满足陈旧配置**重造上游已删除的测试文件**（旧的 + `tests/e2e/online_serving/test_hunyuan_video_15.py` 已由 + `tests/e2e/accuracy/hunyuanvideo15_{i2v,t2v}/` 幸存者取代；恢复旧路径会与幸存者重复、以后每次 rebase 都冲突）；把 rc=4/5 当 OOM 在 GPU 上重试——**结果永远不会改变**，只白烧 3 次重试并硬停；盲目放宽 marker 把 `slow`/`full_model` 幸存者塞进 merge job（改变上游的 CI 成本/分层意图）。 ^[SK-fix-stale-test-path-collection-error-after-rename] diff --git a/knowledge/repos/vllm-omni/rebase/workflow.md b/knowledge/repos/vllm-omni/rebase/workflow.md index 5bf6abb77..b62ab2352 100644 --- a/knowledge/repos/vllm-omni/rebase/workflow.md +++ b/knowledge/repos/vllm-omni/rebase/workflow.md @@ -1,22 +1,22 @@ --- title: "Rebase 工作流：分支、波次与失败路由" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-08-05 type: guide tags: [vllm-omni, rebase] -sources: ["vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh", ".buildkite/rebase-pipeline.yaml"] +sources: ["vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh", ".buildkite/cuda/rebase-pipeline.yml"] --- # Rebase 工作流：分支、波次与失败路由 运营事实来自 rebase-agent 配置快照（@122a9468，**可能漂移**）；仓库侧事实在 -`main @ 5c390096` 复核。运营系统以 rebase-agent 仓库为准，本页是知识树快照 +`v0.26.0 @ a4ea67a2` 复核。运营系统以 rebase-agent 仓库为准，本页是知识树快照 （2026-07-16）。 ## 分支与管线 - 对齐分支 `dev/vllm-align`（目标合回 `main`）；专用 Buildkite 管线 - `.buildkite/rebase-pipeline.yaml`（仓库侧）+ 运营管线 `vllm-omni-rebase` + `.buildkite/cuda/rebase-pipeline.yml`（仓库侧）+ 运营管线 `vllm-omni-rebase` （nightly 与 main CI）、`vllm-omni-release`（CI），org `vllm`；wheel 变体 `cu130`；上次 rebase 的 vLLM 提交 pin `1acd67a795ebccdf9b9db7697ae9082058301657`。 From 13beff7a7396cad5a4d604bbe5ca4fd92f6c6d1c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 11:18:19 +0800 Subject: [PATCH 002/465] knowledge(vllm-omni): sync 76de642c (PR #5701) Upstream-Commit: 76de642c5475732a3351b1c5beb9b86973733688 Upstream-PR: #5701 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 6 +++--- doc/architecture/KNOWLEDGE.md | 4 ++-- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a6094762f..7d16b5c63 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 807db6efd70ff2e9b55a63d6e1b0530e2b74f8f2 - audited_ref: v0.26.0 - audited_sha: a4ea67a21b20054dacc6e83952f9bd407e8ee4e7 + previous_audited_sha: a4ea67a21b20054dacc6e83952f9bd407e8ee4e7 + audited_ref: main + audited_sha: 76de642c5475732a3351b1c5beb9b86973733688 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 21638ad6f..1842b4f40 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -41,8 +41,8 @@ extensions merely because they are absent from the common source. (per-file upstream created/updated dates, captured before the submodule's git history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps - are verified against vllm-omni `v0.26.0 @ - a4ea67a21b20054dacc6e83952f9bd407e8ee4e7`. The canonical machine baseline + are verified against vllm-omni `main @ + 76de642c5475732a3351b1c5beb9b86973733688`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d7080e8d1..95c7bad5e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `v0.26.0 @ a4ea67a2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 76de642c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 42aacd275..472e35189 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `v0.26.0 @ a4ea67a2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 76de642c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 864aac94e..6e9549fbf 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `v0.26.0 @ a4ea67a2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 76de642c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9ef25b665..9cf0d690b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `v0.26.0 @ a4ea67a2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 76de642c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ce5d9b85f..cae2e2853 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `v0.26.0 @ a4ea67a2` 验证存在：`OmniARScheduler`（:50）、 +- 源码校验：以上路径与下列类均已在 `main @ 76de642c` 验证存在：`OmniARScheduler`（:50）、 `OmniARAsyncScheduler`（:928）、`KVCacheTransferData`（:40）、`OmniGenerationScheduler`（:42）、 `OmniSchedulerMixin`（:40）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3f62f7d44..34b685cf0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `v0.26.0 @ a4ea67a2` 验证存在 +- 源码校验：以上路径均已在 `main @ 76de642c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c84dbd8cb..d74455be7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`v0.26.0 @ a4ea67a2`（2026-08-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 76de642c`（2026-08-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 807db6ef \ - --to a4ea67a2 \ + --from a4ea67a2 \ + --to 76de642c \ --repo \ --mode report-only ``` From 46ba0fcf8e4820358e3c7e57c26358c969bbae60 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 11:26:55 +0800 Subject: [PATCH 003/465] knowledge(vllm-omni): sync 7a2007cc (PR #5647) Upstream-Commit: 7a2007cc17d63ce9070e1feefd29fcca8a93cf34 Upstream-PR: #5647 Knowledge-Delta: baseline-pins-and-public-config-migration --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/architecture.md | 17 +++++++++-------- .../config-audit-plain-language.md | 16 +++++++++------- .../components/configuration/deploy-yaml.md | 15 ++++++++------- .../configuration/pipeline-deploy-schema.md | 17 +++++++++-------- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 13 files changed, 47 insertions(+), 42 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7d16b5c63..51e7976a5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a4ea67a21b20054dacc6e83952f9bd407e8ee4e7 + previous_audited_sha: 76de642c5475732a3351b1c5beb9b86973733688 audited_ref: main - audited_sha: 76de642c5475732a3351b1c5beb9b86973733688 + audited_sha: 7a2007cc17d63ce9070e1feefd29fcca8a93cf34 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1842b4f40..0ffe205e5 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 76de642c5475732a3351b1c5beb9b86973733688`. The canonical machine baseline + 7a2007cc17d63ce9070e1feefd29fcca8a93cf34`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 95c7bad5e..ca1316c7b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 76de642c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7a2007cc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/architecture.md b/knowledge/repos/vllm-omni/components/configuration/architecture.md index 5ac9eb022..14d8a1ff5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/architecture.md +++ b/knowledge/repos/vllm-omni/components/configuration/architecture.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 配置构造架构" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-02 type: architecture tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/diffusion/data.py] +sources: ["claude-workflow-starter-private@296ea45", "PR #5647", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/diffusion/data.py, vllm_omni/entrypoints/cli/serve.py] --- # vLLM-Omni 配置构造架构 @@ -90,13 +90,13 @@ structured 与 legacy 可以有不同的最终对象，但不能有不同的字 ## PipelineConfig 与 deploy YAML 的具体结构 -以下事实在 `v0.26.0 @ a4ea67a2` 复核；源码会变化，动手前仍须刷新 live 版本。 +以下事实在 `main @ 7a2007cc` 复核；源码会变化，动手前仍须刷新 live 版本。 - **`PipelineConfig`**（模型的冻结 stage 拓扑）由模型的 `pipeline.py` 注册； **deploy YAML**（`vllm_omni/deploy/*.yaml`）只描述“这些 stage 怎么跑”。 - 未迁移模型仍走 legacy `--stage-configs-path` + `stage_args` schema - （`vllm_omni/model_executor/stage_configs/*.yaml`）。 -- 未显式给 `--deploy-config`/`--stage-configs-path` 时，registry 按 `model_type` + public `vllm serve --omni` 只接受 `--deploy-config`；offline/programmatic API 仍保留 + `stage_configs_path` 读取 legacy `stage_args` schema。 +- serve 未显式给 `--deploy-config` 时，registry 按 `model_type` 自动解析 pipeline + bundled deploy YAML；单 stage diffusion 模型不在 registry， 走 `async_omni_engine.py` 的 `_create_default_diffusion_stage_cfg` 兜底。 - deploy 顶层字段包括 `base_config`、`async_chunk`、`connectors`/`edges`、`stages`、 @@ -129,5 +129,6 @@ out-of-tree 注册。 - `endpoint_policy.py` 的 `OmniServingCapability` 与 `shutdown_unsupported_routes` 允许 pipeline 关闭不支持的 serving 路由。 - `composable_parallel/` 的 `--strategy-config` 在合并后的 stage 上叠加逐 stage - 并行轴；已接线与 reserved axis 必须区分，且不能与 legacy - `--stage-configs-path` 静默混用。 + 并行轴；已接线与 reserved axis 必须区分。Public serve 已移除 legacy flag； + offline/programmatic resolver 仍须拒绝 `stage_configs_path` 与 `deploy_config_path` + 同时出现。 diff --git a/knowledge/repos/vllm-omni/components/configuration/config-audit-plain-language.md b/knowledge/repos/vllm-omni/components/configuration/config-audit-plain-language.md index cc7c89133..f9c3c2487 100644 --- a/knowledge/repos/vllm-omni/components/configuration/config-audit-plain-language.md +++ b/knowledge/repos/vllm-omni/components/configuration/config-audit-plain-language.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni config audit 说人话规则" created: 2026-07-16 -updated: 2026-07-30 +updated: 2026-09-02 type: guide tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45"] +sources: ["claude-workflow-starter-private@296ea45", "PR #5647", vllm_omni/entrypoints/cli/serve.py, examples/online_serving/chart-helm/templates/_helpers.tpl] --- # vLLM-Omni config audit 说人话规则 @@ -83,8 +83,9 @@ CLI / deploy YAML / legacy / default factory ## 围绕五个问题展开 1. 入口太多。 - - 新入口是 `--deploy-config`。 - - 老入口是 `--stage-configs-path`。 + - `vllm serve --omni` 的唯一公开 YAML 入口是 `--deploy-config`。 + - `stage_configs_path` 只保留在 offline/programmatic API 和内部 resolver；serve CLI + 会拒绝 `--stage-configs-path`。 - 还有不传配置时的 diffusion fallback。 2. 默认 diffusion 配置有好几处在造。 @@ -99,9 +100,10 @@ CLI / deploy YAML / legacy / default factory - 所以光看 yaml 不知道最后 runtime 真正用了什么。 4. 新老配置路径混在一起。 - - `--deploy-config` 是新方向。 - - `--stage-configs-path` 还没死。 - - 文档里出现 stage config 不一定都是错的，要区分过时写法和 legacy-required。 + - `--deploy-config` 是 public serve 的唯一方向。 + - `stage_configs_path` 仍存在于 offline/programmatic 调用，不等于 public CLI 仍兼容。 + - 文档里出现 stage config 不一定都是错的：online serve 写法必须迁移，offline + legacy-required 需沿真实入口确认；Helm 旧 key 必须明确报迁移错误，不能静默忽略。 5. 模型迁移和 runtime bugfix 容易混在一起。 - pipeline registry 迁移应该讲 topology。 diff --git a/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md b/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md index 690537341..aeddcd900 100644 --- a/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md +++ b/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md @@ -1,29 +1,30 @@ --- title: "vLLM-Omni deploy YAML 实操" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-02 type: guide tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", vllm_omni/deploy/] +sources: ["claude-workflow-starter-private@296ea45", "PR #5647", vllm_omni/deploy/, vllm_omni/entrypoints/cli/serve.py] --- # Deploy YAML 写作实操 面向"要给模型写/改部署配置"的场景；schema 语义 owner 是 [Configuration](architecture.md)（本页不复制字段表）。 -`v0.26.0 @ a4ea67a2` 复核。 +`main @ 7a2007cc` 复核。 ## 何时需要 YAML，何时 CLI 就够 - bundled 默认：registry 按 `model_type` 自动加载 `vllm_omni/deploy/.yaml` - ——不给 `--deploy-config`/`--stage-configs-path` 时就用它；只调个别 stage 参数时 + ——public `vllm serve --omni` 不给 `--deploy-config` 时就用它；只调个别 stage 参数时 优先 CLI/per-stage override，不新写 YAML。 - 需要新 YAML 的信号：新模型/新 stage 拓扑变体（参照 bagel 的三形态）、平台覆盖 （`platforms: npu/rocm/xpu`）、connector 拓扑改变、或要固化一组经过验证的资源 参数（如 voxcpm2 的 KV pin）。 -- legacy 未迁移模型仍走 `--stage-configs-path` + `stage_args` schema - （`model_executor/stage_configs/*.yaml`，如 mimo_audio、step_audio_2、 - hunyuan_video_15、wan2_2 的 dit_fp8 配置）。 +- public `vllm serve --omni` 已删除 `--stage-configs-path`，在线启动和 Helm 统一使用 + `--deploy-config` / `deployConfigPath`；旧 Helm key 非空时必须 fail fast 并给出替代项。 + Offline/programmatic API 仍可用 `stage_configs_path` 读取 legacy `stage_args` schema + （`model_executor/stage_configs/*.yaml`），不要把 public CLI 删除误写成内部合同已删除。 ## 写作时必查的字段（事故来源） diff --git a/knowledge/repos/vllm-omni/components/configuration/pipeline-deploy-schema.md b/knowledge/repos/vllm-omni/components/configuration/pipeline-deploy-schema.md index dd5c9a49a..3a963aa17 100644 --- a/knowledge/repos/vllm-omni/components/configuration/pipeline-deploy-schema.md +++ b/knowledge/repos/vllm-omni/components/configuration/pipeline-deploy-schema.md @@ -1,15 +1,15 @@ --- title: "PipelineConfig 与 deploy YAML 详细 schema" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-02 type: guide tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_config.py, docs/configuration/stage_configs.md] +sources: ["claude-workflow-starter-private@296ea45", "PR #5647", vllm_omni/config/stage_config.py, vllm_omni/entrypoints/cli/serve.py, docs/configuration/stage_configs.md] --- # PipelineConfig 与 deploy YAML 详细 schema -以下事实在 `v0.26.0 @ a4ea67a2` 复核；当前稳定职责见 +以下事实在 `main @ 7a2007cc` 复核；当前稳定职责见 [Configuration architecture](architecture.md)，官方 spec 见 `docs/configuration/stage_configs.md` （schema 全表）与 `composable_parallel.md`。 @@ -18,9 +18,9 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/stage_conf - **`PipelineConfig`**（模型的冻结 stage 拓扑）由模型的 `pipeline.py` 注册； **deploy YAML**（`vllm_omni/deploy/*.yaml`，79 个）只描述"这些 stage 怎么跑"。 - 未迁移模型仍走 legacy `--stage-configs-path` + `stage_args` schema - （`vllm_omni/model_executor/stage_configs/*.yaml`）。 -- 未显式给 `--deploy-config`/`--stage-configs-path` 时，registry 按 `model_type` + public serve 已删除 `--stage-configs-path`；legacy `stage_args` schema 只通过仍存在的 + offline/programmatic `stage_configs_path` 合同读取。 +- `vllm serve --omni` 未显式给 `--deploy-config` 时，registry 按 `model_type` 自动解析 pipeline + bundled deploy YAML（如 `qwen2_5_omni.yaml` 在 1×H100、 `qwen3_omni_moe.yaml` 在 2×H100 验证过）。 - deploy 顶层字段：`base_config`（overlay 父配置，`stages:`/`platforms:` 按 stage_id @@ -67,7 +67,8 @@ callable（如 `resolve_qwen3_omni_pipeline`）；HF `model_type` 冲突用 - `composable_parallel/`：`--strategy-config` 把逐 stage 并行轴栈 （tp/dp/pp/ep/stage_replica 已接线；sp/cfg/vae_pp/hsdp 等保留位）以声明式 overlay 叠加到合并后的 stage 上、先于 CLI override；显式 deploy 值与 strategy 派生值冲突时 - fail fast，并在 worker 创建前检查 devices 与 `tp × dp × pp` world size；**不能**与 - legacy `--stage-configs-path` 组合。 + fail fast，并在 worker 创建前检查 devices 与 `tp × dp × pp` world size。Public serve + 已没有 legacy CLI 组合；offline/programmatic resolver 同时收到 `stage_configs_path` 和 + `deploy_config_path` 时仍必须拒绝。 源码会变化，具体函数与行号在改代码前必须以目标仓库当前版本为准。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 472e35189..845827057 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 76de642c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 7a2007cc` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6e9549fbf..0a4095544 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 76de642c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7a2007cc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9cf0d690b..a087b4638 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 76de642c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7a2007cc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cae2e2853..b5574cb3b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 76de642c` 验证存在：`OmniARScheduler`（:50）、 +- 源码校验：以上路径与下列类均已在 `main @ 7a2007cc` 验证存在：`OmniARScheduler`（:50）、 `OmniARAsyncScheduler`（:928）、`KVCacheTransferData`（:40）、`OmniGenerationScheduler`（:42）、 `OmniSchedulerMixin`（:40）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 34b685cf0..a984d350a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 76de642c` 验证存在 +- 源码校验：以上路径均已在 `main @ 7a2007cc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d74455be7..28302e090 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 76de642c`（2026-08-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7a2007cc`（2026-08-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from a4ea67a2 \ - --to 76de642c \ + --from 76de642c \ + --to 7a2007cc \ --repo \ --mode report-only ``` From ae9c7eef635de6eeef7cb10b2968d22063e27ab8 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 11:32:00 +0800 Subject: [PATCH 004/465] knowledge(vllm-omni): sync 900a7f08 (PR #5709) Upstream-Commit: 900a7f0813d0482811b0e4dfd3cf7deabbe2429f Upstream-PR: #5709 Knowledge-Delta: baseline-pins-and-minimax-h3-accuracy-oracle --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/minimax-h3/_index.md | 10 +++++++--- 10 files changed, 19 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 51e7976a5..255992326 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 76de642c5475732a3351b1c5beb9b86973733688 + previous_audited_sha: 7a2007cc17d63ce9070e1feefd29fcca8a93cf34 audited_ref: main - audited_sha: 7a2007cc17d63ce9070e1feefd29fcca8a93cf34 + audited_sha: 900a7f0813d0482811b0e4dfd3cf7deabbe2429f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0ffe205e5..c1ebe2759 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 7a2007cc17d63ce9070e1feefd29fcca8a93cf34`. The canonical machine baseline + 900a7f0813d0482811b0e4dfd3cf7deabbe2429f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ca1316c7b..7a913139a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7a2007cc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 900a7f08` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 845827057..c5f0680d4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7a2007cc` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 900a7f08` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0a4095544..892d9a2cd 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7a2007cc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 900a7f08` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a087b4638..80aa70f88 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7a2007cc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 900a7f08` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b5574cb3b..4d89717c6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7a2007cc` 验证存在：`OmniARScheduler`（:50）、 +- 源码校验：以上路径与下列类均已在 `main @ 900a7f08` 验证存在：`OmniARScheduler`（:50）、 `OmniARAsyncScheduler`（:928）、`KVCacheTransferData`（:40）、`OmniGenerationScheduler`（:42）、 `OmniSchedulerMixin`（:40）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a984d350a..21a94265d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7a2007cc` 验证存在 +- 源码校验：以上路径均已在 `main @ 900a7f08` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 28302e090..e611bad2f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7a2007cc`（2026-08-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 900a7f08`（2026-08-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 76de642c \ - --to 7a2007cc \ + --from 7a2007cc \ + --to 900a7f08 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 7bdf663e3..769b98f80 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -1,10 +1,10 @@ --- title: "MiniMax H3" created: 2026-08-05 -updated: 2026-08-05 +updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5709", .buildkite/cuda/test-nightly.yml, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -34,7 +34,11 @@ confidence: high ## 验证入口 -模型专属 contract、packing、parallel 和 e2e 测试在 `tests/diffusion/models/minimax_h3/`。 +模型专属 contract、packing 和 parallel 测试在 `tests/diffusion/models/minimax_h3/`。 +T2VA full-model accuracy 入口在 `tests/e2e/accuracy/minimax_h3/`：它固定模型与官方参考视频 +revision、1344x768/24 FPS/243 帧/50 steps/seed 0，除视频和 AAC 32 kHz stereo metadata +外，还以 SSIM >= 0.82、PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、 +HSDP4、text-encoder TP4 和 VAE patch parallel 4。该用例是精度/媒体合同，不是性能基线。 硬件 recipe 只记录已验证的 GPU/NPU 形状；性能数字不能从 recipe 的配置示例泛化为全硬件 保证。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 [Configuration](../../components/configuration/_index.md) 和 [Serving](../../components/serving/_index.md)。 From 01d1872556133243dc42c1e3d36b4b3a466bc6bc Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 11:38:17 +0800 Subject: [PATCH 005/465] knowledge(vllm-omni): sync 9f9f9f27 (PR #5740) Upstream-Commit: 9f9f9f274aaf7f178abe47e4f9a856c96a1f2b79 Upstream-PR: #5740 Knowledge-Delta: baseline-pins-and-minimax-golden-provenance --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minimax-h3/_index.md | 12 +++++++----- 10 files changed, 19 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 255992326..70d2f917b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7a2007cc17d63ce9070e1feefd29fcca8a93cf34 + previous_audited_sha: 900a7f0813d0482811b0e4dfd3cf7deabbe2429f audited_ref: main - audited_sha: 900a7f0813d0482811b0e4dfd3cf7deabbe2429f + audited_sha: 9f9f9f274aaf7f178abe47e4f9a856c96a1f2b79 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c1ebe2759..e1f5da841 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 900a7f0813d0482811b0e4dfd3cf7deabbe2429f`. The canonical machine baseline + 9f9f9f274aaf7f178abe47e4f9a856c96a1f2b79`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7a913139a..72da3f351 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 900a7f08` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9f9f9f27` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c5f0680d4..11f294033 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 900a7f08` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9f9f9f27` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 892d9a2cd..fb3cd2544 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 900a7f08` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9f9f9f27` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 80aa70f88..f6da893f5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 900a7f08` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9f9f9f27` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4d89717c6..8b0ddd47f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 900a7f08` 验证存在：`OmniARScheduler`（:50）、 +- 源码校验：以上路径与下列类均已在 `main @ 9f9f9f27` 验证存在：`OmniARScheduler`（:50）、 `OmniARAsyncScheduler`（:928）、`KVCacheTransferData`（:40）、`OmniGenerationScheduler`（:42）、 `OmniSchedulerMixin`（:40）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 21a94265d..9d5e396c2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 900a7f08` 验证存在 +- 源码校验：以上路径均已在 `main @ 9f9f9f27` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e611bad2f..478849f68 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 900a7f08`（2026-08-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9f9f9f27`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 7a2007cc \ - --to 900a7f08 \ + --from 900a7f08 \ + --to 9f9f9f27 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 769b98f80..3d9db7776 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5709", .buildkite/cuda/test-nightly.yml, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5709", "PR #5740", .buildkite/cuda/test-nightly.yml, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -35,10 +35,12 @@ confidence: high ## 验证入口 模型专属 contract、packing 和 parallel 测试在 `tests/diffusion/models/minimax_h3/`。 -T2VA full-model accuracy 入口在 `tests/e2e/accuracy/minimax_h3/`：它固定模型与官方参考视频 -revision、1344x768/24 FPS/243 帧/50 steps/seed 0，除视频和 AAC 32 kHz stereo metadata -外，还以 SSIM >= 0.82、PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、 -HSDP4、text-encoder TP4 和 VAE patch parallel 4。该用例是精度/媒体合同，不是性能基线。 +T2VA full-model accuracy 入口在 `tests/e2e/accuracy/minimax_h3/`：模型 snapshot 固定在 +`73372e6c`，但该 revision 没有参考视频，official `assets/t2va.mp4` 实际从 Hugging Face +`resolve/main` 下载，不能把 golden 写成 revision-immutable。用例固定 1344x768/24 FPS/ +243 帧/50 steps/seed 0，除视频和 AAC 32 kHz stereo metadata 外，还以 SSIM >= 0.82、 +PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、text-encoder TP4 +和 VAE patch parallel 4。该用例是精度/媒体合同，不是性能基线。 硬件 recipe 只记录已验证的 GPU/NPU 形状；性能数字不能从 recipe 的配置示例泛化为全硬件 保证。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 [Configuration](../../components/configuration/_index.md) 和 [Serving](../../components/serving/_index.md)。 From bd0c8d0287fbb749e79a2a485184132ee3253093 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 11:42:23 +0800 Subject: [PATCH 006/465] knowledge(vllm-omni): sync 30cba842 (PR #5715) Upstream-Commit: 30cba84222e24653b444d70ed440aec00b964e41 Upstream-PR: #5715 Knowledge-Delta: baseline-pins-and-release-install-contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/docs/_index.md | 15 +++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 25 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 70d2f917b..8c83d3c27 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 900a7f0813d0482811b0e4dfd3cf7deabbe2429f + previous_audited_sha: 9f9f9f274aaf7f178abe47e4f9a856c96a1f2b79 audited_ref: main - audited_sha: 9f9f9f274aaf7f178abe47e4f9a856c96a1f2b79 + audited_sha: 30cba84222e24653b444d70ed440aec00b964e41 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e1f5da841..c9b96fe5c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 9f9f9f274aaf7f178abe47e4f9a856c96a1f2b79`. The canonical machine baseline + 30cba84222e24653b444d70ed440aec00b964e41`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 72da3f351..a7280680e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9f9f9f27` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 30cba842` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 11f294033..df3e376af 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9f9f9f27` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 30cba842` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index fb3cd2544..5cc41b9ae 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9f9f9f27` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 30cba842` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f6da893f5..00f300448 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9f9f9f27` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 30cba842` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8b0ddd47f..cc50e68a0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9f9f9f27` 验证存在：`OmniARScheduler`（:50）、 +- 源码校验：以上路径与下列类均已在 `main @ 30cba842` 验证存在：`OmniARScheduler`（:50）、 `OmniARAsyncScheduler`（:928）、`KVCacheTransferData`（:40）、`OmniGenerationScheduler`（:42）、 `OmniSchedulerMixin`（:40）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9d5e396c2..c2e557126 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9f9f9f27` 验证存在 +- 源码校验：以上路径均已在 `main @ 30cba842` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/_index.md b/knowledge/repos/vllm-omni/docs/_index.md index 1dc85a502..5e5e76ff7 100644 --- a/knowledge/repos/vllm-omni/docs/_index.md +++ b/knowledge/repos/vllm-omni/docs/_index.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 文档" created: 2026-07-10 -updated: 2026-07-16 +updated: 2026-09-02 type: index tags: [vllm-omni, docs] -sources: [] +sources: ["PR #5715", README.md, docs/configuration/README.md, docs/getting_started/installation/README.md, docs/getting_started/installation/gpu/cuda.inc.md, docs/getting_started/installation/npu/npu.inc.md] --- # vLLM-Omni 文档 @@ -23,3 +23,14 @@ sources: [] |---|---| | 判断 RFC 是否仍在进行 | [RFC status](rfcs/_index.md) | | 找上游官方设计文档与其知识树 owner | [design-doc map](design-doc-map.md) | + +## 发布与安装边界 + +- stable release 从 0.14 起跟随 upstream vLLM 的偶数 minor cadence；版本兼容按 matching + major/minor 判断。`vLLM-Omni 0.26.x` 要求 `vLLM 0.26.x`，稳定 0.26.0 文档固定 + vLLM `v0.26.0`，configuration 链接也必须指向 0.26 文档，不能拿旧版本示例混用。 +- CUDA 文档的 0.26 base image 是 `vllm/vllm-openai:v0.26.0`；NPU source install 另有 + 平台专属 tuple：vLLM `v0.26.0` + vLLM-Ascend `releases/v0.26.0rc`。Review 明确没有把 + NPU 更新扩散到 MUSA/quickstart，因此各平台 pin 必须回到自己的安装页核对。 +- README 将 full-duplex realtime serving 标为 experimental；公开说明、验收和支持承诺 + 不能把它写成 stable feature。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 478849f68..c95b8d68c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9f9f9f27`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 30cba842`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 900a7f08 \ - --to 9f9f9f27 \ + --from 9f9f9f27 \ + --to 30cba842 \ --repo \ --mode report-only ``` From 28650943697f37d1d6626b86bfffc6840afcd936 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 11:48:01 +0800 Subject: [PATCH 007/465] knowledge(vllm-omni): sync d5d47e9b (PR #5610) Upstream-Commit: d5d47e9be12dc973c0083228476e963ab2e98797 Upstream-PR: #5610 Knowledge-Delta: baseline-pins-and-async-output-contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/architecture.md | 19 +++++++++++++-- .../components/model-executor/rules.md | 23 ++++++++++++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/docs/design-doc-map.md | 9 ++++---- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 12 files changed, 56 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8c83d3c27..3ed380132 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9f9f9f274aaf7f178abe47e4f9a856c96a1f2b79 + previous_audited_sha: 30cba84222e24653b444d70ed440aec00b964e41 audited_ref: main - audited_sha: 30cba84222e24653b444d70ed440aec00b964e41 + audited_sha: d5d47e9be12dc973c0083228476e963ab2e98797 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c9b96fe5c..b2ec9b5d4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 30cba84222e24653b444d70ed440aec00b964e41`. The canonical machine baseline + d5d47e9be12dc973c0083228476e963ab2e98797`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a7280680e..5372fcc45 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 30cba842` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d5d47e9b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index df3e376af..168f36ff8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 30cba842` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d5d47e9b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5cc41b9ae..fa8c3574b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 30cba842` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d5d47e9b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 00f300448..23b59cc27 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 30cba842` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d5d47e9b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/architecture.md b/knowledge/repos/vllm-omni/components/model-executor/architecture.md index aa19679cf..9ca8ba019 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/architecture.md +++ b/knowledge/repos/vllm-omni/components/model-executor/architecture.md @@ -1,10 +1,10 @@ --- title: "Model Executor 共享架构" created: 2026-07-10 -updated: 2026-08-05 +updated: 2026-09-02 type: architecture tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/worker/test_omni_gpu_model_runner.py] +sources: ["PR #5610", docs/design/feature/omni_async_output_materialization.md, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] --- # Model Executor 共享架构 @@ -35,6 +35,21 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/config/stage_config.py 启动类错误优先沿“最终 stage 配置 → runtime devices → 启动前校验 → worker rank”读取这四段。只有其中一段把错误状态交给其他模块时才横向展开。 +## AR async Omni output materialization + +`GPUARModelRunner` 可把 `OmniModelRunnerOutput` 的 D2H、逐请求切片和 payload 构造从 +decode critical path 移到 background builder；采样 token feedback 与下一步要读的 +postprocess state 仍须同步完成。进入下一 decode step 前，scheduler/request mapping、token +span 和可复用 CUDA output 都要形成 step-owned snapshot，CUDA tensor 要 clone 后在独立 +stream 搬到 pinned host buffer，并用 event 标记 ready。 + +snapshot 不包含 connector output：background builder 是该 output cycle 的唯一 drain +consumer，读取 live connector signal；`get_output()` join builder，同时是完成与异常传播边界。 +启用条件是 AR async scheduling、`async_chunk`、model opt-in 且没有 prefix cache、speculative +decode、routed-expert output 等冲突状态；条件不满足时回退同步构造。CUDA/ROCm 是已验证范围， +不是代码里的 platform guard；XPU/MUSA 可能通过共享 GPU runner 进入但尚未验证，Ascend NPU +使用独立 runner 并保持同步 materialization。 + ## 怎样判断问题归属 多个模型共用的 stage 生命周期、配置解析、runner 预处理合同或数据桥接问题归这里；某个模型怎样解读 metadata、token、attention 或 checkpoint 的问题归对应 `models/<模型>/`。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 8180db260..1adbebbc5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-08-23 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, "PR #3422", "PR #3642", "PR #4730", "PR #5074", "PR #5792", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, "PR #3422", "PR #3642", "PR #4730", "PR #5074", "PR #5610", "PR #5792", "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -21,6 +21,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | stage TP/PP/DP、devices、replica、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | +| async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -28,6 +29,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | | `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`, `EXEC-1b`, `EXEC-1c` | | `loader-contract` | dtype、checkpoint config 获取、loader | `EXEC-2a` | +| `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | ## 严格配置校验 @@ -87,6 +89,25 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru - 平台边界：查 GPU generation/AR runner 和 NPU 等平台是继承共享实现还是覆盖它；没有 live 继承或调用链证据时，不得声称所有平台合同已经一致。 - 文档与兼容：已向 out-of-tree 模型开放的字段必须从 model-contribution 入口可发现，并明确旧 runtime 和新 consumer 需要怎样配套升级；不能用某个 in-tree 模型的 fallback 充当公开合同。 +## Async Omni output materialization + +### EXEC-5a — snapshot、live drain 与 fallback 必须构成同一个 output-cycle 合同 + +- 触发：修改 `GPUARModelRunner` async output、background builder、D2H copy、connector + output、model opt-in/postprocess，或 prefix cache/spec decode/routed-expert/platform guard。 +- 强制：下一 decode step 会覆盖或 mutate 的 scheduler metadata、request mapping、token span + 和 CUDA output 必须先做 step-owned snapshot；可复用 tensor 要 clone，并由独立 stream、 + pinned host buffer 和 ready event 固定 D2H 生命周期。采样 token feedback 与下一步所需的 + model postprocess state 保持 eager。未 snapshot 的 connector signal 必须只有 background + builder 一个 drain consumer；`get_output()` 必须 join builder 并传播 background exception。 +- 禁止：把 live connector state 写进 snapshot 清单；让同步路径和 background path 重复 drain； + 把“CUDA/ROCm 已验证”写成不存在的 platform guard；在 prefix cache、speculative decode、 + routed-expert output 或非 eager stateful postprocess 下强开异步路径。 +- 验收：覆盖 reusable buffer 被下一 step 覆盖、逐请求 snapshot 隔离、connector drain 次数/ + 顺序、builder exception 和每个 compatibility guard 的 synchronous fallback。平台结论按真实 + runner 分开：CUDA/ROCm 已验证，XPU/MUSA 可能进入共享 GPU path 但未验证，Ascend NPU 的 + 独立 runner 仍同步 materialize。 ^[PR #5610] + ## Stage 并行度和设备容量必须一起验收 - 触发条件：修改或排查全局 CLI、per-stage override、deploy YAML、平台 overlay、stage 并行度、`runtime.devices`、设备映射或 worker 启动。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cc50e68a0..311dd6fcc 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 30cba842` 验证存在：`OmniARScheduler`（:50）、 +- 源码校验：以上路径与下列类均已在 `main @ d5d47e9b` 验证存在：`OmniARScheduler`（:50）、 `OmniARAsyncScheduler`（:928）、`KVCacheTransferData`（:40）、`OmniGenerationScheduler`（:42）、 `OmniSchedulerMixin`（:40）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c2e557126..3db7228c7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 30cba842` 验证存在 +- 源码校验：以上路径均已在 `main @ d5d47e9b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index f02a2fd4f..8823cd029 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -1,16 +1,16 @@ --- title: "官方设计文档地图（docs/design/**）" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-02 type: guide tags: [vllm-omni, docs] -sources: [docs/design/index.md, docs/design/architecture_overview.md] +sources: [docs/design/index.md, docs/design/architecture_overview.md, docs/design/feature/omni_async_output_materialization.md] --- # 官方设计文档地图（docs/design/**） -`v0.26.0 @ a4ea67a2` 复核。注意：官方 `docs/design/index.md` 只列了子集（Architecture -Overview、5 篇 feature、metrics、3 篇 module）；**完整树比索引大得多**—— +`main @ d5d47e9b` 复核。注意：官方 `docs/design/index.md` 只列了子集（Architecture +Overview、6 篇 feature、metrics、3 篇 module）；**完整树比索引大得多**—— `feature/` 下还有 async_chunk、cache_dit、teacache、prefix_caching、7 篇并行策略、 `omni_connectors/` 逐后端 spec 等未入索引的文档，找 spec 时直接 `ls docs/design/feature/`。 @@ -19,6 +19,7 @@ Overview、5 篇 feature、metrics、3 篇 module）；**完整树比索引大 | `architecture_overview.md` | 三类模型拓扑（DiT 主/AR 主/AR+DiT）、OmniRouter/EntryPoints/AR/Diffusion/OmniConnector、E/P/D/G 解耦、CFG companion 流 | [serving](../components/serving/architecture.md) + [distributed](../components/distributed/architecture.md) | | `module/async_omni_architecture.md` | 五层运行时（API→Engine→Orchestration→Communication→Execution），ZMQ + janus 队列，Qwen3-Omni worked example | [serving](../components/serving/architecture.md)、[qwen-omni](../models/qwen-omni/architecture.md) | | `module/ar_module.md` | AR 模块继承链与请求流转 | [scheduler](../components/scheduler/architecture.md) + [model-executor](../components/model-executor/architecture.md) | +| `feature/omni_async_output_materialization.md` | AR decode 关键路径外的异步 D2H 与 Omni payload 构造；含 live connector ownership、平台验证范围和同步 fallback | [model-executor](../components/model-executor/_index.md) | | `module/dit_module.md` | Diffusion 引擎/调度/worker/pipeline/加速组件 | [diffusion](../components/diffusion/architecture.md) | | `module/entrypoint_module.md` | **stub（"update soon"）——上游文档缺口** | [serving](../components/serving/_index.md) | | `feature/disaggregated_inference.md` + `omni_connectors/*` | connector 选择矩阵与逐后端 spec | [distributed](../components/distributed/connector-backends.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c95b8d68c..739091527 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 30cba842`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d5d47e9b`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 9f9f9f27 \ - --to 30cba842 \ + --from 30cba842 \ + --to d5d47e9b \ --repo \ --mode report-only ``` From acde0113630e8a854020b4eab24f1b2ec82f11d4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 11:55:30 +0800 Subject: [PATCH 008/465] knowledge(vllm-omni): sync 5215e03a (PR #5461) Upstream-Commit: 5215e03a91adecbb5ffece29aa74360a7569d0c5 Upstream-PR: #5461 Knowledge-Delta: baseline-pins-and-shared-scheduler-lifecycle --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 13 ++++---- .../components/scheduler/architecture.md | 31 +++++++++++-------- .../vllm-omni/components/scheduler/rules.md | 16 +++++++--- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 11 files changed, 48 insertions(+), 34 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3ed380132..23ba0e0ac 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 30cba84222e24653b444d70ed440aec00b964e41 + previous_audited_sha: d5d47e9be12dc973c0083228476e963ab2e98797 audited_ref: main - audited_sha: d5d47e9be12dc973c0083228476e963ab2e98797 + audited_sha: 5215e03a91adecbb5ffece29aa74360a7569d0c5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b2ec9b5d4..2ce50b97d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - d5d47e9be12dc973c0083228476e963ab2e98797`. The canonical machine baseline + 5215e03a91adecbb5ffece29aa74360a7569d0c5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5372fcc45..d022d85e5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d5d47e9b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5215e03a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 168f36ff8..6e1b4df01 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d5d47e9b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5215e03a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index fa8c3574b..d8e0921ae 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d5d47e9b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5215e03a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 23b59cc27..f106e75a9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d5d47e9b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5215e03a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 311dd6fcc..fcae2f718 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -1,19 +1,19 @@ --- title: "Scheduler（AR/生成请求调度）" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-02 type: index tags: [vllm-omni, components, scheduler] -sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache.py, docs/design/module/ar_module.md] +sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, vllm_omni/core/prefix_cache.py, docs/design/module/ar_module.md] --- # Scheduler（AR/生成请求调度） - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d5d47e9b` 验证存在：`OmniARScheduler`（:50）、 - `OmniARAsyncScheduler`（:928）、`KVCacheTransferData`（:40）、`OmniGenerationScheduler`（:42）、 - `OmniSchedulerMixin`（:40）、`OmniTensorPrefixCache`（prefix_cache.py:33） +- 源码校验：以上路径与下列类均已在 `main @ 5215e03a` 验证存在：`OmniARScheduler`（:73）、 + `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 + `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） - 测试入口：`tests/core/` - 主要职责：AR/生成 stage 的请求调度（继承 vLLM Scheduler）、跨 stage KV transfer 的调度面、 @@ -22,7 +22,8 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/prefix_cache ## 什么时候查这里 - 调查请求调度、waiting/running 状态转换、`WAITING_FOR_CHUNK`/`WAITING_FOR_INPUT`。 -- 排查跨 stage KV transfer 的调度侧（`KVCacheTransferData`、kv_ready）。 +- 排查跨 stage KV transfer 的调度侧（transfer criteria、request lifecycle、kv_ready）； + serialized `KVCacheTransferData` 本体属于 [Distributed](../distributed/_index.md)。 - 排查 `OmniTensorPrefixCache` 引起的跨 stage payload 截断或缓存 miss。 ## 不放什么 diff --git a/knowledge/repos/vllm-omni/components/scheduler/architecture.md b/knowledge/repos/vllm-omni/components/scheduler/architecture.md index e358daeef..256c09fcb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/architecture.md +++ b/knowledge/repos/vllm-omni/components/scheduler/architecture.md @@ -1,35 +1,40 @@ --- title: "Scheduler 共享架构" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-02 type: architecture tags: [vllm-omni, components, scheduler] -sources: [docs/design/module/ar_module.md, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/prefix_cache.py, vllm_omni/worker/gpu_ar_model_runner.py] +sources: [docs/design/module/ar_module.md, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/prefix_cache.py, vllm_omni/worker/gpu_ar_model_runner.py] --- # Scheduler 共享架构 -以下事实在 `v0.26.0 @ a4ea67a2` 复核；官方叙述见 +以下事实在 `main @ 5215e03a` 复核；官方叙述见 `docs/design/module/ar_module.md`（含继承 classDiagram 与请求流转 flowchart）。 ## 继承链（对 vLLM 的扩展方式） -- `OmniARScheduler(OmniSchedulerMixin, VLLMScheduler)` —— `vllm_omni/core/sched/omni_ar_scheduler.py:50`。 - `schedule()` 调 `super().schedule()` 后，把 base `NewRequestData` 重包成 - `OmniNewRequestData`，附上 `prompt_embeds` 与 `additional_information`（跨 stage - 载荷）；`update_from_output()` 保持 vLLM 原语义。异步变体 - `OmniARAsyncScheduler(OmniARScheduler, AsyncVLLMScheduler)`（:928）。 +- `OmniARScheduler(OmniSchedulerMixin, VLLMScheduler)` —— `vllm_omni/core/sched/omni_ar_scheduler.py:73`。 + `schedule()` 调 `super().schedule()` 后，通过 `OmniNewRequestData.from_base()` 保留 base + dataclass 的每个字段，再附加 `external_req_id`、`additional_information` 与 + `model_intermediate_buffer`；已经是 Omni entry 的 fast path 不重建。异步变体 + `OmniARAsyncScheduler(OmniARScheduler, AsyncVLLMScheduler)`（:815）。 - `OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler)` - （`omni_generation_scheduler.py:42`）—— 非 AR/单步架构（Conv/LSTM、code2wav 等）的 + （`omni_generation_scheduler.py:29`）—— 非 AR/单步架构（Conv/LSTM、code2wav 等）的 快路径：`schedule()` 一次性为请求分配全部输入 token（预算不足回退默认调度）； `update_from_output()` 单步后直接置 `FINISHED_STOPPED` 并 `_free_request`。 -- `OmniSchedulerMixin`（`omni_scheduler_mixin.py:40`）承载两者共享的 omni 调度行为。 +- `OmniSchedulerMixin` 承载两者共享的 Omni I/O 生命周期：初始化 chunk/full-payload + 协调状态、消费 pending input、恢复临时停放队列、无损重包 `NewRequestData`、清理完成 + 请求，以及汇总 KV stats/events、finished sets 和 scheduler stats。AR 与 generation + scheduler 仍各自拥有调度策略和 finished-ID 语义；AR 的 synthetic-abort output 是显式 + 本地策略，不能被共享 helper 隐式扩散。 ## 跨 stage KV transfer（调度面） -`KVCacheTransferData`（`omni_ar_scheduler.py:40`）是 AR scheduler 携带的跨 stage KV -迁移合同；调度器决定"何时可发/何时已就绪"，实际搬运由 -[Distributed 组件](../distributed/_index.md)的 connector/KV-transfer 管理面执行。 +AR scheduler 的 transfer criteria 与 request lifecycle 决定“何时可发/何时已就绪”； +serialized `KVCacheTransferData` 位于 +`distributed/omni_connectors/kv_transfer_manager.py:146`，实际搬运由 +[Distributed 组件](../distributed/_index.md)的 connector/KV-transfer 数据面执行。 真实案例（只链接不复制）： [HunyuanImage3 KV reuse 事故](../../models/hunyuan-image3/incidents/2026-05-13-kv-reuse-orchestrator.md) 中 `omni_ar_scheduler.py` 的 kv_ready 发射与 `_mark_request_for_kv_transfer` 是根因链的一环。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 81d5f3bae..2a7cf9aa6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -1,10 +1,10 @@ --- title: "Scheduler 规则" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-02 type: rule tags: [vllm-omni, components, scheduler] -sources: ["vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, "PR #4106"] +sources: ["vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5461"] --- # Scheduler 规则 @@ -126,8 +126,16 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 `update_from_output`/异步调度、kv-connector 统计等挂点）是否有签名、时序或语义 变化，逐条登记后再改代码；曾发生 `kv_connector_stats` 提取时序在上游更新后错位、 需要移到 `_update_from_kv_xfer_finished` 之后的案例（omni_ar_scheduler.py 与 - omni_generation_scheduler.py 都要改）。 -- 禁止：只跑单测绿灯就认定调度语义未变（单测常用 `object.__new__` 绕过真实构造）。 + omni_generation_scheduler.py 都要改）。base `NewRequestData` 重包必须按 live dataclass + 字段无损复制后再附 Omni payload；共享 mixin 只集中机械 lifecycle，AR/generation 的 + admission、cached-payload 和 synthetic-abort 等差异用调用参数保留为显式本地策略。 +- 禁止：只跑单测绿灯就认定调度语义未变；不能用缺少真实 mixin helper 的 + `SimpleNamespace`/`object.__new__` stub 绕过共享调用链，也不能手列一份会随 upstream + schema 漂移的 base output/request 字段。 +- 验收：枚举 live `NewRequestData.__dataclass_fields__` 逐字段断言 identity/equality，覆盖 + Omni fast path 不重建与 generation fallback；AR/generation 分别验证 pending-input、queue + restore、finished-set/abort policy。直接调用 scheduler 方法的轻量 stub 必须继承 mixin 或 + 显式绑定本次方法真实依赖，使断言确实到达目标分支。 ^[PR #5461] ## SCHED-4a — side-stream 复制必须拥有源 buffer 的完成期 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3db7228c7..b492fa142 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d5d47e9b` 验证存在 +- 源码校验：以上路径均已在 `main @ 5215e03a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 739091527..8287822b3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d5d47e9b`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5215e03a`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 30cba842 \ - --to d5d47e9b \ + --from d5d47e9b \ + --to 5215e03a \ --repo \ --mode report-only ``` From 8a428d7df6620fd9f74e9ea4683cd2e1e9742a16 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:02:56 +0800 Subject: [PATCH 009/465] knowledge(vllm-omni): sync b9a51592 (PR #5737) Upstream-Commit: b9a51592f0ca845e23dad9018f25e9fda4b4928c Upstream-PR: #5737 Knowledge-Delta: baseline-pins-and-minimax-h3-fp8-contract --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 27 ++++++---- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../vllm-omni/models/minimax-h3/_index.md | 16 +++++- .../vllm-omni/models/minimax-h3/rules.md | 54 +++++++++++++++++++ 12 files changed, 97 insertions(+), 24 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 23ba0e0ac..60c80a022 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d5d47e9be12dc973c0083228476e963ab2e98797 + previous_audited_sha: 5215e03a91adecbb5ffece29aa74360a7569d0c5 audited_ref: main - audited_sha: 5215e03a91adecbb5ffece29aa74360a7569d0c5 + audited_sha: b9a51592f0ca845e23dad9018f25e9fda4b4928c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2ce50b97d..2d20302c4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 5215e03a91adecbb5ffece29aa74360a7569d0c5`. The canonical machine baseline + b9a51592f0ca845e23dad9018f25e9fda4b4928c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d022d85e5..71124d37c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5215e03a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b9a51592` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6e1b4df01..154572fe2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5215e03a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b9a51592` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index d490f3219..bc45488ae 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-08-23 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5737", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/] confidence: high --- @@ -89,12 +89,14 @@ confidence: high - 触发：增加或修改 weight mapper、scale 名称、quantization adapter 或 key resolution。 - 强制：从序列化 key 追到目标 layer 注册的 parameter/buffer 和 forward consumer； - 多条 resolution path 必须返回对称、由合同解释的目标名。 + 多条 resolution path 必须返回对称、由合同解释的目标名。模型专属 grouped-QKV reorder、 + fused gate/up split 等布局变换必须在调用 active weight loader 前完成，让 quantization wrapper + 与 TP shard 接收最终语义布局。 - 禁止：把 producer 有而当前 consumer 不支持的 tensor 静默过滤；必须 fold/map 或 fail fast，并在错误中标明依赖的 upstream 版本边界。 - 验收：测试覆盖已消费 key、未知 key、当前版本不支持的 scale 以及两条 resolution path 的输出名；fused mixed-FP8 还须覆盖 weight/scale/shard、gate/up 顺序、nested serialized - FP8 和 combined projection，缺 shard 不得默认 0。 ^[PR #5087] ^[PR #5848] + FP8 和 combined projection，缺 shard 不得默认 0。 ^[PR #5087] ^[PR #5737] ^[PR #5848] ### DIFF-2b — HSDP/FSDP 修复必须执行真实 fully_shard @@ -105,18 +107,21 @@ confidence: high identity，并覆盖 loader 的真实调用边界；replicate size 非正值拒绝，1 用 1D、>1 用 2D DeviceMesh，测试 world size 与参数一致。 ^[PR #5088] ^[PR #5872] -### DIFF-2c — component quantization 独立解析且保留完整 owner 前缀 +### DIFF-2c — component quantization 独立解析且 namespace 端到端一致 - 触发：diffusion pipeline 为 text encoder、transformer、VAE 等组件增加独立量化配置。 -- 强制：每个组件独立解析量化配置，并把包含 component owner 的完整模块名前缀传到 - 真正持有权重的 layer；只量化明确支持的 attention/MLP linear，embedding、LM head - 等排除项必须显式保持未量化。 -- 禁止：先裁掉 `text_encoder` 等 owner 前缀再匹配 component 规则；用一个组件的配置 - 隐式覆盖其他组件；因为同属一个模型就量化全部 linear。 +- 强制：每个组件独立解析量化配置，并选择一个端到端一致的 namespace：要么 selector + 保留 `text_encoder` 等完整 component owner prefix 到持权 layer，要么 pipeline 先 resolve + 单一 component，再让内部 layer 使用 component-relative prefix。只量化明确支持的 linear， + embedding、LM head、patch/final projection 等排除项必须显式保持未量化。 +- 禁止：半途裁掉 owner prefix；resolve component 后又要求 ignored layer 带 owner；用一个 + 组件配置隐式覆盖其他组件；因为同属一个模型就量化全部 linear。 - 验收：至少覆盖“只量化一个组件、其他组件保持 BF16”的真实构造与加载，逐层断言 命中/排除集合，并验证 meta-device parameter 不会被提前 move。FLUX.2 的具体边界见 - [FLUX.2 规则](../../models/flux2/rules.md)；新增 quantization×offload 组合需给兼容矩阵， - 未验证 quantizer fail-closed，并核对 dtype/shape/stride。 ^[PR #5136] ^[PR #6279] + [FLUX.2 规则](../../models/flux2/rules.md)，component-relative 变体见 + [MiniMax H3 规则](../../models/minimax-h3/rules.md)；新增 quantization×offload 组合需给兼容矩阵， + 未验证 quantizer fail-closed，并核对 dtype/shape/stride。 ^[PR #5136] ^[PR #5737] + ^[PR #6279] ### DIFF-2d — Artifact identity 只信任已验证的不可变来源 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d8e0921ae..79415cb1a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5215e03a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b9a51592` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f106e75a9..8e31e7b3a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5215e03a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b9a51592` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index fcae2f718..042acccd1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5215e03a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b9a51592` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b492fa142..ffbc63e9a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5215e03a` 验证存在 +- 源码校验：以上路径均已在 `main @ b9a51592` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8287822b3..8e81dac6b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5215e03a`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b9a51592`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from d5d47e9b \ - --to 5215e03a \ + --from 5215e03a \ + --to b9a51592 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 3d9db7776..cef82c6c3 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5709", "PR #5740", .buildkite/cuda/test-nightly.yml, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5709", "PR #5737", "PR #5740", .buildkite/cuda/test-nightly.yml, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -28,6 +28,15 @@ confidence: high 并且必须同时整除 Qwen3-VL 的 64 个 attention heads 与 8 个 KV heads。 - 单卡 accuracy 路径使用 CPU offload；多卡部署可使用 Ulysses、text-encoder TP、VAE tile/patch parallel 或 layerwise offload，但每个组合都必须按最终并行拓扑和设备数验证。 +- H3 DiT 支持加载时 online FP8：默认覆盖 token-refiner/主 block 的 attention、MLP、 + condition projection 和 AdaLN projection；video/audio patch、timestep MLP、最终输出头、 + text encoder 与 VAE 保持 BF16/FP32。`ignored_layers` 按 DiT 内部的精确 runtime prefix + 匹配，不带 `transformer.` owner prefix。checkpoint 的 grouped-QKV reorder 和 fused-MLP + gate/up split 在 `MiniMaxH3DiTModel.load_weights()` 内先执行，再交给当前 vLLM loader， + 以保留 TP shard 与 FP8 online-processing wrapper。 +- H3 online FP8 当前不得与 layerwise offload 组合：offload 产生的 weight stride + 会被 Cutlass FP8 kernel 拒绝。变更量化覆盖范围时，除逐层命中/排除与加载转换测试外， + 必须同时保护 joint video/audio 质量；peak memory 只作为同 case report，不是稳定上界。 - 音频加载优先使用 torchaudio；当 TorchCodec/torchaudio 在 CPU-only aarch64 环境不可用 时，`reference_video.load_audio_file` 回退到 soundfile，再对 libsndfile 不支持的格式 通过 ffmpeg 转 WAV。该回退保持 `(channels, samples)` float32 与原始 sample rate 合同。 @@ -44,3 +53,8 @@ PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、t 硬件 recipe 只记录已验证的 GPU/NPU 形状；性能数字不能从 recipe 的配置示例泛化为全硬件 保证。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 [Configuration](../../components/configuration/_index.md) 和 [Serving](../../components/serving/_index.md)。 + +## 审查入口 + +H3 online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 +[MiniMax H3 rules](rules.md#direct-代码快速入口)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md new file mode 100644 index 000000000..d90b79b32 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -0,0 +1,54 @@ +--- +title: "MiniMax H3 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5737", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, recipes/MiniMaxAI/MiniMax-H3.md] +confidence: high +--- + +# MiniMax H3 规则 + +只有 `MMH3-数字字母` 是可审计规则 ID。 + +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | 第一批源码 | +|---|---|---| +| online FP8、`ignored_layers`、component prefix | `MMH3-1a` | `pipeline_minimax_h3.py::_resolve_component_quant_config` → `MiniMaxH3DiTModel` linear prefix | +| grouped QKV、fused MLP、weight loader、TP | `MMH3-1a` | `minimax_h3_transformer.py::MiniMaxH3DiTModel.load_weights` → active vLLM loader | +| FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | + +## MMH3-1a — component namespace 与 checkpoint transform 必须在 active loader 前闭合 + +- 触发：修改 H3 online FP8 覆盖范围、`ignored_layers`、linear prefix、QKV/MLP checkpoint + 变换或 TP loader。 +- 强制：pipeline 先把 structured quantization config 解析到 `transformer` component，再构造 + DiT；因此内部 prefix 是 `blocks.*`、`token_refiner.*` 等 component-relative 名称，不带 + `transformer.`。每个 eligible linear 显式传 prefix/quant config，patch、timestep、final + projection 显式保持非量化。grouped-QKV reorder 与 fused `fc1` gate/up split 在 + `MiniMaxH3DiTModel.load_weights()` 先完成，再调用当前 parameter 的 active weight loader， + 让 TP shard 与 FP8 `online_process_loader` 看到最终布局。 +- 禁止：用 parent prefix 代替 exact leaf `ignored_layers`；一边已 resolve component、一边仍 + 要求 `transformer.`；在 linear 构造后包掉 active loader，或把 reorder/split 放到 FP8 + wrapper 之后。 +- 验收：枚举默认 quantized/full-precision prefix 集合，逐个验证 exact ignored leaf；QKV + 与 gate/up sentinel 断言传给 active loader 的顺序和 shard id，至少覆盖 TP 与 online FP8。 + ^[PR #5737] + +## MMH3-1b — joint video/audio quality 与 offload 兼容性必须一起验收 + +- 触发：改变 H3 FP8 layer scope、kernel/loader、quality threshold、offload 或 nightly lane。 +- 强制：H3 online FP8 与 layerwise offload 当前标为 incompatible；resident FP8 可与 TP、VAE + tiling 组合。BF16/FP8 A/B 使用同一 immutable checkpoint、prompt、seed、size、frames、 + steps 与并行/资源设置；joint output 同时检查 video LPIPS 和 32 kHz audio 的 spectral + cosine/RMS ratio。PSNR、MAE 与 peak memory 是 report-only 指标，不能冒充 gate。 +- 禁止：只验视频就宣称 H3 joint-output quality；把一次 22% memory observation 写成稳定 + 上界；让 FP8+layerwise offload 到 Cutlass kernel 才因 flattened-weight stride 失败。 +- 验收：当前 2x H100 case 的 gates 是 LPIPS <= 0.20、audio spectral cosine >= 0.80、 + RMS ratio 在 0.50–2.00，且 sample rate 为 32000；单测还要证明 phase-tolerant metric + 接受相移但拒绝 spectral drift。 ^[PR #5737] + +共享 component quantization、checkpoint mapping 与 quality evidence 见 +[Diffusion rules](../../components/diffusion/rules.md)。 From bd26a6e007ba5607bc243ef2aaa452b96f3542b7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:12:37 +0800 Subject: [PATCH 010/465] knowledge(vllm-omni): sync b6728db4 (PR #5756) Upstream-Commit: b6728db486a31d5e86fa167484f49e638357a430 Upstream-PR: #5756 Knowledge-Delta: baseline-pins-and-comfyui-reference-routing --- adapters/vllm_omni/release_baseline.yaml | 6 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/_index.md | 3 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../vllm-omni/models/minimax-h3/_index.md | 8 +++- knowledge/repos/vllm-omni/tooling/_index.md | 17 ++++++++ knowledge/repos/vllm-omni/tooling/rules.md | 43 +++++++++++++++++++ 13 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 knowledge/repos/vllm-omni/tooling/_index.md create mode 100644 knowledge/repos/vllm-omni/tooling/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 60c80a022..664d9b50d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5215e03a91adecbb5ffece29aa74360a7569d0c5 + previous_audited_sha: b9a51592f0ca845e23dad9018f25e9fda4b4928c audited_ref: main - audited_sha: b9a51592f0ca845e23dad9018f25e9fda4b4928c + audited_sha: b6728db486a31d5e86fa167484f49e638357a430 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -105,7 +105,7 @@ owner_documents: testing: - knowledge/repos/vllm-omni/ci/_index.md tooling: - - knowledge/repos/vllm-omni/rules.md + - knowledge/repos/vllm-omni/tooling/rules.md runtime-core: - knowledge/repos/vllm-omni/components/model-executor/rules.md diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2d20302c4..c2a35c428 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - b9a51592f0ca845e23dad9018f25e9fda4b4928c`. The canonical machine baseline + b6728db486a31d5e86fa167484f49e638357a430`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/_index.md b/knowledge/repos/vllm-omni/_index.md index f672625cf..05832dc4b 100644 --- a/knowledge/repos/vllm-omni/_index.md +++ b/knowledge/repos/vllm-omni/_index.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni 入口" created: 2026-07-10 -updated: 2026-08-23 +updated: 2026-09-02 type: index tags: [vllm-omni] sources: [] @@ -42,5 +42,6 @@ owner 仍不明确时才看 [components 职责表](components/_index.md)；模 | bug 和行为异常 | [debug](debug/_index.md) | | Git、PR、rebase | [git](git/_index.md)、[upstream rebase](rebase/_index.md) | | benchmark / profiling | [benchmark](benchmark/_index.md) | +| ComfyUI、示例 app 和客户端适配 | [tooling](tooling/_index.md) | | 远端验证 | [remote](remote/_index.md) | | 关键模型与负责人 | [owners](owners/_index.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 71124d37c..02a609b47 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b9a51592` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b6728db4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 154572fe2..e6e40359f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b9a51592` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b6728db4` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 79415cb1a..cca5e9044 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b9a51592` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b6728db4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8e31e7b3a..894f35d55 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b9a51592` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b6728db4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 042acccd1..0252dab15 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b9a51592` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b6728db4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ffbc63e9a..c391d3294 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b9a51592` 验证存在 +- 源码校验：以上路径均已在 `main @ b6728db4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8e81dac6b..a32e92dc8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b9a51592`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b6728db4`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 5215e03a \ - --to b9a51592 \ + --from b9a51592 \ + --to b6728db4 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index cef82c6c3..b3aca020e 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5709", "PR #5737", "PR #5740", .buildkite/cuda/test-nightly.yml, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5709", "PR #5737", "PR #5740", "PR #5756", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -41,6 +41,12 @@ confidence: high 时，`reference_video.load_audio_file` 回退到 soundfile，再对 libsndfile 不支持的格式 通过 ffmpeg 转 WAV。该回退保持 `(channels, samples)` float32 与原始 sample rate 合同。 +## ComfyUI 请求路由 + +MiniMax H3 的 `t2va`、`fl2va` 和 `ref2va` frontend 选择、reference multipart 字段及 +Hub/local model lookup 合同由 [ComfyUI tooling rules](../../tooling/rules.md) 负责；模型页不复制 +客户端规则。 + ## 验证入口 模型专属 contract、packing 和 parallel 测试在 `tests/diffusion/models/minimax_h3/`。 diff --git a/knowledge/repos/vllm-omni/tooling/_index.md b/knowledge/repos/vllm-omni/tooling/_index.md new file mode 100644 index 000000000..9390ee354 --- /dev/null +++ b/knowledge/repos/vllm-omni/tooling/_index.md @@ -0,0 +1,17 @@ +--- +title: "vLLM-Omni apps 与 tooling" +created: 2026-09-02 +updated: 2026-09-02 +type: index +tags: [vllm-omni] +sources: [apps/ComfyUI-vLLM-Omni/] +--- + +# Apps 与 tooling + +本 owner 负责仓库内示例 app、外部 UI adapter 和它们到 vLLM-Omni 公开接口的客户端合同； +服务端 HTTP/OpenAI handler 仍归 [Serving](../components/serving/_index.md)。 + +| 遇到什么 | 查看哪里 | +|---|---| +| ComfyUI node 输入、multipart payload、model spec 或参数 builder | [tooling rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/tooling/rules.md b/knowledge/repos/vllm-omni/tooling/rules.md new file mode 100644 index 000000000..5eeec929a --- /dev/null +++ b/knowledge/repos/vllm-omni/tooling/rules.md @@ -0,0 +1,43 @@ +--- +title: "vLLM-Omni apps 与 tooling 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni] +sources: ["PR #5756", apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py, tests/e2e/features/comfyui/test_comfyui_integration.py] +confidence: high +--- + +# Apps 与 tooling 规则 + +只有 `OMNI-TOOL-数字字母` 是可审计规则 ID。 + +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | 第一批源码 | +|---|---|---| +| ComfyUI video node、frame/reference、multipart payload | `OMNI-TOOL-1a` | `nodes.py` → `utils/types.py` → `utils/api_client.py` | +| ComfyUI model path、partition、params builder | `OMNI-TOOL-1b` | `utils/models.py` → `utils/api_client.py` → integration test | + +## OMNI-TOOL-1a — reference 输入状态与 multipart 字段必须一起验证 + +- 触发：修改 ComfyUI video node 的 `frame`、`references`、task 选择或上传字段。 +- 强制：无输入路由到 `t2va`，仅 `frame` 路由到 `fl2va`，仅 `references` 路由到 + `ref2va`，并在发请求前拒绝 frame/reference 同时存在。Ref2VA 只接受“一张图像 + 一段 + 音频”或“一到两个纯视频”；不得把 video 与 image/audio reference 混用。 +- payload：frame/image 使用 `input_reference`，audio 使用 JSON `audio_reference`，每段 + video 分别追加一个 `input_references`；model task 必须通过 `extra_params` 到达参数 builder。 +- 验收：公开 node 到 API client 的测试分别覆盖三条 task 分支、互斥/数量错误，以及最终 + `multimodal_data` 和请求参数；只测 validator helper 不算端到端证据。 ^[PR #5756] + +## OMNI-TOOL-1b — model lookup 不得丢失本地路径中的 partition + +- 触发：增加 model family、`/FL2VA` 或 `/Ref2VA` partition，或修改参数 builder dispatch。 +- 强制：lookup 同时支持 Hub ID 和任意父目录下以 `MiniMax-H3/FL2VA|Ref2VA` 结尾的本地 + 路径；归一化路径时必须保留 family/partition 后缀，不能只取 basename。builder 调用必须 + 使用其 keyword-only `extra_params` 合同。 +- MiniMax H3 的 `flow_shift` 保持顶层字段，`audio_flow_shift` 和 `task` 放在 + `extra_params`；内部 `type` 只在 model dispatch 完成后移除。 +- 验收：Hub ID、本地绝对路径及两个 partition 都命中同一 model spec，并断言 builder 收到 + 正确 keyword 和最终字段层级。PR review 曾直接发现 basename lookup 与错误 builder keyword， + 因此这两项必须作为回归 fence。 ^[PR #5756] From 93c334b47b991d3bf1b54eb46e6bfff6401976d1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:22:13 +0800 Subject: [PATCH 011/465] knowledge(vllm-omni): sync 6fdc4ca6 (PR #4958) Upstream-Commit: 6fdc4ca62dc6080bc683552c6ec9e310e9c80867 Upstream-PR: #4958 Knowledge-Delta: baseline-pins-and-fused-code-predictor-loader --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/architecture.md | 13 ++++++++++- .../components/model-executor/rules.md | 23 ++++++++++++++++--- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/moss-tts/architecture.md | 12 +++++++--- .../vllm-omni/models/qwen3-tts/_index.md | 6 +++-- 13 files changed, 57 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 664d9b50d..95edefd4e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b9a51592f0ca845e23dad9018f25e9fda4b4928c + previous_audited_sha: b6728db486a31d5e86fa167484f49e638357a430 audited_ref: main - audited_sha: b6728db486a31d5e86fa167484f49e638357a430 + audited_sha: 6fdc4ca62dc6080bc683552c6ec9e310e9c80867 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c2a35c428..08ecc77dd 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - b6728db486a31d5e86fa167484f49e638357a430`. The canonical machine baseline + 6fdc4ca62dc6080bc683552c6ec9e310e9c80867`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 02a609b47..406d97b3b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b6728db4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6fdc4ca6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e6e40359f..b7512d643 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b6728db4` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6fdc4ca6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index cca5e9044..4abbecf48 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b6728db4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6fdc4ca6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 894f35d55..82b85cadb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b6728db4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6fdc4ca6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/architecture.md b/knowledge/repos/vllm-omni/components/model-executor/architecture.md index 9ca8ba019..c6e1fcb45 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/architecture.md +++ b/knowledge/repos/vllm-omni/components/model-executor/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: architecture tags: [vllm-omni, components, model-executor] -sources: ["PR #5610", docs/design/feature/omni_async_output_materialization.md, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] +sources: ["PR #4958", "PR #5610", docs/design/feature/omni_async_output_materialization.md, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/model_executor/models/moss_tts/test_moss_fused_load.py, tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] --- # Model Executor 共享架构 @@ -50,6 +50,17 @@ decode、routed-expert output 等冲突状态；条件不满足时回退同步 不是代码里的 platform guard；XPU/MUSA 可能通过共享 GPU runner 进入但尚未验证，Ascend NPU 使用独立 runner 并保持同步 materialization。 +## 共享 fused code-predictor loader 合同 + +Qwen3-TTS、Qwen3-Omni 和 MOSS-TTS Realtime 复用 `qwen3_code_predictor.py`。每层将 +HF q/k/v 行按该顺序拼成 plain `nn.Linear` `qkv_proj`，将 gate/up 拼成 +`gate_up_proj`；forward 必须按同样边界拆分，并在 SDPA 前把 packed k/v slice +重新物化为 contiguous。这保持线性代数，但 GPU kernel/累加顺序变化仍可导致 +bit-level drift。 + +该表示的 loader、consumer、TP 与 platform 可执行验收合同见 +[EXEC-2b](rules.md#exec-2b-fused-shard-必须按布局数值闭环且所有-consumer-委托共享-loader)。 + ## 怎样判断问题归属 多个模型共用的 stage 生命周期、配置解析、runner 预处理合同或数据桥接问题归这里；某个模型怎样解读 metadata、token、attention 或 checkpoint 的问题归对应 `models/<模型>/`。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 1adbebbc5..a86e4871c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -1,10 +1,10 @@ --- title: "Model Executor 规则" created: 2026-07-10 -updated: 2026-08-23 +updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, "PR #3422", "PR #3642", "PR #4730", "PR #5074", "PR #5610", "PR #5792", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, tests/model_executor/models/moss_tts/test_moss_fused_load.py, tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5074", "PR #5610", "PR #5792", "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -21,6 +21,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | stage TP/PP/DP、devices、replica、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | +| fused projection、HF shard 拼装、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | `models/common/qwen3_code_predictor.py::CodePredictorBaseModel.load_weights` → Qwen3/MOSS consumer → platform override | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | 审查组 | 什么时候触发 | 规则 ID | @@ -28,7 +29,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `core` | 每次 model-executor 审查 | `EXEC-1a` | | `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | | `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`, `EXEC-1b`, `EXEC-1c` | -| `loader-contract` | dtype、checkpoint config 获取、loader | `EXEC-2a` | +| `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`, `EXEC-2b` | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | @@ -165,3 +166,19 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 验收：mock 下载层证明只请求目标 config，loader 测试断言各子模块 dtype；真实 smoke 记录峰值显存和 dtype。Krea 2 的具体约束见 [Krea 2 规则](../../models/krea2/rules.md)。 ^[PR #4730] + +### EXEC-2b — fused shard 必须按布局数值闭环且所有 consumer 委托共享 loader + +- 触发：合并/拆分 q/k/v、gate/up 等 projection，修改 HF shard 映射、wrapper/talker + loader、packed projection TP plan 或平台 override。 +- 强制：weight 与可选 bias 都按 forward split 的同一顺序数值拼装；部分 shard 和 + 整组 shard 缺失都硬失败。所有声明同一 fused module 的 consumer 必须委托给共享 + loader，wrapper 并恢复 returned loaded-name 的模型前缀；平台 override 同步使用新 + fused 属性与共享 split helper。 +- 禁止：不得用逐 tensor `default_weight_loader` 绕过 fused assembler；不得只记录 skipped + shard 后让随机初始参数继续；不得只用 shape 或 returned-name 断言顺序正确。GQA + 下错误偏移仍可形状合法。plain packed `nn.Linear` 不得声明泛化 colwise TP； + 未有 TP-aware packing/loading/split 与 TP=2 测试时，TP plan 保持空。 +- 验收：数值比较 fused 参数与 `cat([q,k,v])`/`cat([gate,up])`，覆盖 bias、 + 部分/整组缺 shard、每个 consumer 的委托和前缀；以非等 q/KV width 证明 split 能识别 + GQA 错序。平台测试或静态 guard 证明不再引用已删除的 projection 属性。 ^[PR #4958] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0252dab15..efc8e4fca 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b6728db4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6fdc4ca6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c391d3294..cc8d80ca7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b6728db4` 验证存在 +- 源码校验：以上路径均已在 `main @ 6fdc4ca6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a32e92dc8..c54657991 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b6728db4`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6fdc4ca6`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from b9a51592 \ - --to b6728db4 \ + --from b6728db4 \ + --to 6fdc4ca6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md index ae8b3cc79..050fb38ea 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md @@ -1,15 +1,15 @@ --- title: "MOSS-TTS 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: architecture tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, vllm_omni/model_executor/models/moss_tts/pipeline.py, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/moss_voice_generator.yaml] +sources: ["PR #4958", vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, vllm_omni/model_executor/models/moss_tts/pipeline.py, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/moss_voice_generator.yaml] --- # MOSS-TTS 架构 -事实在 `main @ 5d44868e` 复核;变体/deploy 速查见 [index](_index.md)。 +事实在 `main @ 6fdc4ca6` 复核;变体/deploy 速查见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -36,6 +36,12 @@ sources: [vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, - CUDA-graph 安全不对称：8B delay talker 可开 stage-0 graph;VoiceGenerator 必须 eager(每请求 GPU 缓冲不 graph 安全);codec graph 只在 `moss_tts.yaml` 开。 +- Realtime 的 local depth transformer 复用共享 `CodePredictorBaseModel`，因此 checkpoint 中 + q/k/v 和 gate/up shard 不能经由 talker 的逐 tensor `default_weight_loader`。talker 必须将 + `local_transformer.model.*` 委托给 `MossTTSRealtimeLocalTransformer.load_weights()`，由共享 + fused loader 完成拼装并检查完整性。否则 unfused shard 只会被记为 skipped，fused + 参数保持随机初始值却不报错。拼装、缺 shard、returned prefix 和 TP 的共享验收合同见 + [Model Executor EXEC-2b](../../components/model-executor/rules.md#exec-2b-fused-shard-必须按布局数值闭环且所有-consumer-委托共享-loader)。 ## 从输入到输出的主要流程 diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md index e46cf6315..d9b4f1a5b 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md @@ -1,10 +1,10 @@ --- title: "Qwen3-TTS" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-02 type: index tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", vllm_omni/entrypoints/openai/serving_speech.py] +sources: ["PR #4958", "PR #5157", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/common/qwen3_code_predictor.py] confidence: high --- @@ -19,5 +19,7 @@ confidence: high ## 什么时候查这里 - 审查 `x_vector_only_mode`、ICL、`ref_audio` artifact-only reuse 或 engine 存活性。 +- 审查 Qwen3-TTS/Qwen3-Omni 共用 code predictor 的 fused projection、checkpoint 拼装、 + TP 边界或平台 override 时，转到 [Model Executor EXEC-2b](../../components/model-executor/rules.md#exec-2b-fused-shard-必须按布局数值闭环且所有-consumer-委托共享-loader)。 - 描述直达源码与具体不变量见 [Qwen3-TTS rules](rules.md#direct-代码快速入口)；模型家族结构见 [Qwen-Omni](../qwen-omni/_index.md)。 From 4a2fa1ff2989691c1a253da946e84cf801fcef4f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:29:14 +0800 Subject: [PATCH 012/465] knowledge(vllm-omni): sync 1c4aca7f (PR #5744) Upstream-Commit: 1c4aca7f6e3bd24507872e193e091a6af55140f6 Upstream-PR: #5744 Knowledge-Delta: baseline-pins-and-payload-connector-ownership --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/distributed/rules.md | 18 ++++++++++++++++-- .../components/model-executor/_index.md | 2 +- .../components/model-executor/architecture.md | 10 +++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 37 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 95edefd4e..3025b3b75 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b6728db486a31d5e86fa167484f49e638357a430 + previous_audited_sha: 6fdc4ca62dc6080bc683552c6ec9e310e9c80867 audited_ref: main - audited_sha: 6fdc4ca62dc6080bc683552c6ec9e310e9c80867 + audited_sha: 1c4aca7f6e3bd24507872e193e091a6af55140f6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 08ecc77dd..a72ca7223 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 6fdc4ca62dc6080bc683552c6ec9e310e9c80867`. The canonical machine baseline + 1c4aca7f6e3bd24507872e193e091a6af55140f6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 406d97b3b..e777743b2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6fdc4ca6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1c4aca7f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b7512d643..10e6dd282 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6fdc4ca6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1c4aca7f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4abbecf48..df4bf71c3 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6fdc4ca6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1c4aca7f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index b028b8ba4..9e53b4b90 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -1,10 +1,10 @@ --- title: "Distributed 传输规则" created: 2026-08-05 -updated: 2026-08-05 +updated: 2026-09-02 type: rule tags: [vllm-omni, components, distributed] -sources: [vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py] +sources: ["PR #5744", vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py] confidence: high --- @@ -34,3 +34,17 @@ confidence: high connector 没有新数据时无限占用 active window 或静默丢掉 terminal update。 - 验收：覆盖非空→空→非空、upstream exhaustion、stage completion、重复 terminal update 和窗口恢复；首批测试看 `test_chunk_transfer_adapter.py`。 + +## DIST-1c — payload connector 按 edge 所有权创建且不与 KV manager 捆绑 + +- 触发：修改 AR/DiT 解耦、stage connector role、`custom_process_next_stage_input_func`、 + runner connector 初始化或 KV-only edge。 +- 强制：receiver/非 sender 即使无 downstream hook 也创建 payload connector；sender 只在 + `custom_process_next_stage_input_func` 是非空字符串时创建，空值或非字符串都表示 + KV manager 独自拥有该 edge。KV manager 与 payload connector 必须独立初始化和关闭。 +- 禁止：不得只因已配 connector/role 就创建 payload transport；不得从“无 outgoing + hook”推导 receiver 也不需要 connector；不得用 consumer-side hook 反推 sender 所有权。 +- 验收：参数化锁定 sender+空/非字符串→不创建、sender+非空字符串→创建、 + receiver/非 sender+无 hook→创建，并覆盖安全 shutdown。Bagel/Hunyuan KV-only 和 + Qwen3-Omni/MiniCPM-o payload 路径各做真实拓扑 smoke。该矩阵不证明 issue #5595 中独立的 + shutdown/orphan 问题已解决。 ^[PR #5744] diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 82b85cadb..86a84d4d2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6fdc4ca6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1c4aca7f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/architecture.md b/knowledge/repos/vllm-omni/components/model-executor/architecture.md index c6e1fcb45..d91eb96cf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/architecture.md +++ b/knowledge/repos/vllm-omni/components/model-executor/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: architecture tags: [vllm-omni, components, model-executor] -sources: ["PR #4958", "PR #5610", docs/design/feature/omni_async_output_materialization.md, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/model_executor/models/moss_tts/test_moss_fused_load.py, tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] +sources: ["PR #4958", "PR #5610", "PR #5744", docs/design/feature/omni_async_output_materialization.md, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/model_executor/models/moss_tts/test_moss_fused_load.py, tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py, tests/worker/test_omni_connector_mixin.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] --- # Model Executor 共享架构 @@ -50,6 +50,14 @@ decode、routed-expert output 等冲突状态；条件不满足时回退同步 不是代码里的 platform guard；XPU/MUSA 可能通过共享 GPU runner 进入但尚未验证，Ascend NPU 使用独立 runner 并保持同步 materialization。 +## Payload 与 KV connector 所有权 + +`OmniConnectorModelRunnerMixin` 把 runner payload 传输与 `OmniKVTransferManager` 管理的 KV +传输视为独立数据面。receiver 总是创建 payload connector；sender 只在声明了非空 +`custom_process_next_stage_input_func` 时才拥有 payload connector。因此 Bagel/Hunyuan 这类 +KV-only sender 可只保留 KV manager，不另外创建无 consumer 的 payload transport。 +可执行验收见 [Distributed DIST-1c](../distributed/rules.md#dist-1c-payload-connector-按-edge-所有权创建且不与-kv-manager-捆绑)。 + ## 共享 fused code-predictor loader 合同 Qwen3-TTS、Qwen3-Omni 和 MOSS-TTS Realtime 复用 `qwen3_code_predictor.py`。每层将 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index efc8e4fca..c2c220dbe 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6fdc4ca6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1c4aca7f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cc8d80ca7..c0255b652 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6fdc4ca6` 验证存在 +- 源码校验：以上路径均已在 `main @ 1c4aca7f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c54657991..9aa2d9b82 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6fdc4ca6`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1c4aca7f`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from b6728db4 \ - --to 6fdc4ca6 \ + --from 6fdc4ca6 \ + --to 1c4aca7f \ --repo \ --mode report-only ``` From d001f08817aac1c718de86dcc292e0e3f3ba309d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:36:33 +0800 Subject: [PATCH 013/465] knowledge(vllm-omni): sync 78c144f3 (PR #5777) Upstream-Commit: 78c144f3a8f1e4fb3e9d9e0c38bc0a0e635c7c98 Upstream-PR: #5777 Knowledge-Delta: baseline-pins-and-unfused-code-predictor-revert --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/architecture.md | 17 +++++++---------- .../components/model-executor/rules.md | 14 +++++++++----- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/moss-tts/architecture.md | 10 ++-------- .../repos/vllm-omni/models/qwen3-tts/_index.md | 4 +--- 13 files changed, 31 insertions(+), 38 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3025b3b75..310954d97 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6fdc4ca62dc6080bc683552c6ec9e310e9c80867 + previous_audited_sha: 1c4aca7f6e3bd24507872e193e091a6af55140f6 audited_ref: main - audited_sha: 1c4aca7f6e3bd24507872e193e091a6af55140f6 + audited_sha: 78c144f3a8f1e4fb3e9d9e0c38bc0a0e635c7c98 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a72ca7223..4ff5a53bc 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 1c4aca7f6e3bd24507872e193e091a6af55140f6`. The canonical machine baseline + 78c144f3a8f1e4fb3e9d9e0c38bc0a0e635c7c98`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e777743b2..7fc7662a3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1c4aca7f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 78c144f3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 10e6dd282..46ef576e0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1c4aca7f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 78c144f3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index df4bf71c3..dce546872 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1c4aca7f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 78c144f3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 86a84d4d2..4cb9701d7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1c4aca7f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 78c144f3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/architecture.md b/knowledge/repos/vllm-omni/components/model-executor/architecture.md index d91eb96cf..b85513ab8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/architecture.md +++ b/knowledge/repos/vllm-omni/components/model-executor/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: architecture tags: [vllm-omni, components, model-executor] -sources: ["PR #4958", "PR #5610", "PR #5744", docs/design/feature/omni_async_output_materialization.md, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/model_executor/models/moss_tts/test_moss_fused_load.py, tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py, tests/worker/test_omni_connector_mixin.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] +sources: ["PR #5610", "PR #5744", docs/design/feature/omni_async_output_materialization.md, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/worker/test_omni_connector_mixin.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] --- # Model Executor 共享架构 @@ -58,16 +58,13 @@ decode、routed-expert output 等冲突状态；条件不满足时回退同步 KV-only sender 可只保留 KV manager，不另外创建无 consumer 的 payload transport。 可执行验收见 [Distributed DIST-1c](../distributed/rules.md#dist-1c-payload-connector-按-edge-所有权创建且不与-kv-manager-捆绑)。 -## 共享 fused code-predictor loader 合同 +## Qwen3 code predictor 当前投影布局 -Qwen3-TTS、Qwen3-Omni 和 MOSS-TTS Realtime 复用 `qwen3_code_predictor.py`。每层将 -HF q/k/v 行按该顺序拼成 plain `nn.Linear` `qkv_proj`，将 gate/up 拼成 -`gate_up_proj`；forward 必须按同样边界拆分，并在 SDPA 前把 packed k/v slice -重新物化为 contiguous。这保持线性代数，但 GPU kernel/累加顺序变化仍可导致 -bit-level drift。 - -该表示的 loader、consumer、TP 与 platform 可执行验收合同见 -[EXEC-2b](rules.md#exec-2b-fused-shard-必须按布局数值闭环且所有-consumer-委托共享-loader)。 +`main @ 78c144f3` 的共享 code predictor 保持分离 `q_proj`/`k_proj`/`v_proj` +和 `gate_proj`/`up_proj`，HF 同名参数由普通 loader 直接写入，TP plan 也指向这些 +分离名称。只有 310P 平台 overlay 在 `prepare_qkv_weights()` 内为本地执行临时拼接 +QKV weight/bias；这不表示共享模型或 checkpoint loader 已改为 fused parameter。未来再引入 +共享 fusion 时的验收门禁见 [EXEC-2b](rules.md#exec-2b-未来引入-fused-shard-时必须按布局数值闭环)。 ## 怎样判断问题归属 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index a86e4871c..7821ecbe7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, tests/model_executor/models/moss_tts/test_moss_fused_load.py, tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5074", "PR #5610", "PR #5792", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5074", "PR #5610", "PR #5777", "PR #5792", "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -21,7 +21,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | stage TP/PP/DP、devices、replica、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | -| fused projection、HF shard 拼装、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | `models/common/qwen3_code_predictor.py::CodePredictorBaseModel.load_weights` → Qwen3/MOSS consumer → platform override | +| 准备引入 fused projection、HF shard 拼装、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | 先确认目标 main 是否已有 fused parameter；当前 Qwen3 code predictor 为分离 projection | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | 审查组 | 什么时候触发 | 规则 ID | @@ -167,9 +167,13 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 记录峰值显存和 dtype。Krea 2 的具体约束见 [Krea 2 规则](../../models/krea2/rules.md)。 ^[PR #4730] -### EXEC-2b — fused shard 必须按布局数值闭环且所有 consumer 委托共享 loader +### EXEC-2b — 未来引入 fused shard 时必须按布局数值闭环 -- 触发：合并/拆分 q/k/v、gate/up 等 projection，修改 HF shard 映射、wrapper/talker +> 当前 `main @ 78c144f3` 的 Qwen3 code predictor 使用分离 q/k/v 和 gate/up projection； +> PR #4958 的 fused 实现已因 CI failure 被 PR #5777 整体回退。本规则是对未来再引入 +> 类似优化的门禁，不描述当前 runtime 机制。 + +- 触发：准备合并 q/k/v、gate/up 等 projection，修改 HF shard 映射、wrapper/talker loader、packed projection TP plan 或平台 override。 - 强制：weight 与可选 bias 都按 forward split 的同一顺序数值拼装；部分 shard 和 整组 shard 缺失都硬失败。所有声明同一 fused module 的 consumer 必须委托给共享 @@ -181,4 +185,4 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 未有 TP-aware packing/loading/split 与 TP=2 测试时，TP plan 保持空。 - 验收：数值比较 fused 参数与 `cat([q,k,v])`/`cat([gate,up])`，覆盖 bias、 部分/整组缺 shard、每个 consumer 的委托和前缀；以非等 q/KV width 证明 split 能识别 - GQA 错序。平台测试或静态 guard 证明不再引用已删除的 projection 属性。 ^[PR #4958] + GQA 错序。平台测试或静态 guard 证明不再引用已删除的 projection 属性。 ^[PR #4958] ^[PR #5777] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c2c220dbe..9773f78a2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1c4aca7f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 78c144f3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c0255b652..f94f8d869 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1c4aca7f` 验证存在 +- 源码校验：以上路径均已在 `main @ 78c144f3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9aa2d9b82..67e8fc59b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1c4aca7f`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 78c144f3`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 6fdc4ca6 \ - --to 1c4aca7f \ + --from 1c4aca7f \ + --to 78c144f3 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md index 050fb38ea..ff6194d5e 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md @@ -4,12 +4,12 @@ created: 2026-07-21 updated: 2026-09-02 type: architecture tags: [vllm-omni, models] -sources: ["PR #4958", vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, vllm_omni/model_executor/models/moss_tts/pipeline.py, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/moss_voice_generator.yaml] +sources: [vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, vllm_omni/model_executor/models/moss_tts/pipeline.py, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/moss_voice_generator.yaml] --- # MOSS-TTS 架构 -事实在 `main @ 6fdc4ca6` 复核;变体/deploy 速查见 [index](_index.md)。 +事实在 `main @ 78c144f3` 复核;变体/deploy 速查见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -36,12 +36,6 @@ sources: ["PR #4958", vllm_omni/model_executor/models/common/qwen3_code_predicto - CUDA-graph 安全不对称：8B delay talker 可开 stage-0 graph;VoiceGenerator 必须 eager(每请求 GPU 缓冲不 graph 安全);codec graph 只在 `moss_tts.yaml` 开。 -- Realtime 的 local depth transformer 复用共享 `CodePredictorBaseModel`，因此 checkpoint 中 - q/k/v 和 gate/up shard 不能经由 talker 的逐 tensor `default_weight_loader`。talker 必须将 - `local_transformer.model.*` 委托给 `MossTTSRealtimeLocalTransformer.load_weights()`，由共享 - fused loader 完成拼装并检查完整性。否则 unfused shard 只会被记为 skipped，fused - 参数保持随机初始值却不报错。拼装、缺 shard、returned prefix 和 TP 的共享验收合同见 - [Model Executor EXEC-2b](../../components/model-executor/rules.md#exec-2b-fused-shard-必须按布局数值闭环且所有-consumer-委托共享-loader)。 ## 从输入到输出的主要流程 diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md index d9b4f1a5b..06c704cf0 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #4958", "PR #5157", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/common/qwen3_code_predictor.py] +sources: ["PR #5157", vllm_omni/entrypoints/openai/serving_speech.py] confidence: high --- @@ -19,7 +19,5 @@ confidence: high ## 什么时候查这里 - 审查 `x_vector_only_mode`、ICL、`ref_audio` artifact-only reuse 或 engine 存活性。 -- 审查 Qwen3-TTS/Qwen3-Omni 共用 code predictor 的 fused projection、checkpoint 拼装、 - TP 边界或平台 override 时，转到 [Model Executor EXEC-2b](../../components/model-executor/rules.md#exec-2b-fused-shard-必须按布局数值闭环且所有-consumer-委托共享-loader)。 - 描述直达源码与具体不变量见 [Qwen3-TTS rules](rules.md#direct-代码快速入口)；模型家族结构见 [Qwen-Omni](../qwen-omni/_index.md)。 From d784adf0e45f3da0ec9fd0c28ab6be8e2bfe9a0c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:41:37 +0800 Subject: [PATCH 014/465] knowledge(vllm-omni): sync fbea7886 (PR #5085) Upstream-Commit: fbea7886682e332c0450c5e4d0c75b330b661166 Upstream-PR: #5085 Knowledge-Delta: baseline-pins-and-video-loader-contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../components/serving/architecture.md | 10 ++++++++-- .../vllm-omni/components/serving/rules.md | 20 +++++++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 38 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 310954d97..49c9e337d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1c4aca7f6e3bd24507872e193e091a6af55140f6 + previous_audited_sha: 78c144f3a8f1e4fb3e9d9e0c38bc0a0e635c7c98 audited_ref: main - audited_sha: 78c144f3a8f1e4fb3e9d9e0c38bc0a0e635c7c98 + audited_sha: fbea7886682e332c0450c5e4d0c75b330b661166 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4ff5a53bc..e5a9361b6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 78c144f3a8f1e4fb3e9d9e0c38bc0a0e635c7c98`. The canonical machine baseline + fbea7886682e332c0450c5e4d0c75b330b661166`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7fc7662a3..60732324f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 78c144f3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fbea7886` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 46ef576e0..57f8e9981 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 78c144f3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ fbea7886` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index dce546872..34cdeec95 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 78c144f3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fbea7886` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4cb9701d7..743bc5d84 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 78c144f3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fbea7886` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9773f78a2..bfc79511f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 78c144f3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fbea7886` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f94f8d869..4c3ca23bd 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 78c144f3` 验证存在 +- 源码校验：以上路径均已在 `main @ fbea7886` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/architecture.md b/knowledge/repos/vllm-omni/components/serving/architecture.md index 6b2ebd13b..c942a64b0 100644 --- a/knowledge/repos/vllm-omni/components/serving/architecture.md +++ b/knowledge/repos/vllm-omni/components/serving/architecture.md @@ -1,10 +1,10 @@ --- title: "Serving 共享架构" created: 2026-07-10 -updated: 2026-07-23 +updated: 2026-09-02 type: architecture tags: [vllm-omni, components, serving] -sources: [vllm_omni/entrypoints/, vllm_omni/engine/orchestrator.py] +sources: ["PR #5085", vllm_omni/entrypoints/, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, tests/entrypoints/openai_api/test_video_server.py] --- # Serving 共享架构 @@ -25,6 +25,12 @@ Serving 层把用户输入转换成内部请求，选择 online/offline 执行 Serving 只公开有明确请求语义和下游 consumer 的字段。Sampling dataclass、engine state 或 pipeline state 可以包含 tensor、KV 状态和运行时中间量，但这些内部字段不会因此自动成为请求字段。 +Video reference 字节在 `video_api_utils.py` 通过 vLLM `VIDEO_LOADER_REGISTRY` 的 +`omni` backend 解码。该 backend 只定义顺序 first-N/last-N 帧索引，`_decode_video_bytes` +将 `max_frames` 映射为 `num_frames`，从 loader metadata 恢复 fps，并将数组转为 RGB +PIL frames。当前 loader 调用固定 `backend="pyav"`；注册表复用不等于用户已能 +选择 opencv/torchcodec 等 decoder。 + ## 主要源码和调用入口 - `vllm_omni/entrypoints/openai/`：OpenAI-compatible HTTP 请求解析和响应。 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 9faeb0310..7c2b5dcfb 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -1,10 +1,10 @@ --- title: "Serving 规则" created: 2026-07-20 -updated: 2026-08-23 +updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5157", "PR #5670", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py] +sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -25,6 +25,7 @@ confidence: high | sleep/wake、partial stage/tag、idempotency、ACK、generation admission | `engine-lifecycle`：`SERV-5a`, `SERV-5b` | `entrypoints/async_omni.py::{AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni.generate}` → `worker/base.py::{handle_sleep_task,handle_wake_task}` / `diffusion/worker/diffusion_worker.py` | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | +| video reference 解码、frame selection、decoder backend、空帧 | `media-ingress`：`SERV-1c` | `entrypoints/openai/video_api_utils.py::{OmniVideoBackend,_decode_video_bytes}` → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | | Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle | `metrics-lifecycle`：`SERV-2a`, `SERV-2b` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | @@ -32,6 +33,7 @@ confidence: high |---|---|---| | `core` | 每次 serving 审查 | `SERV-4c` | | `streaming-format` | SSE、audio streaming、format/default/capability | `SERV-1a`, `SERV-1b` | +| `media-ingress` | video reference、decoder registry/backend、first/last frames | `SERV-1c` | | `metrics-lifecycle` | metrics、gauge、replica、collector | `SERV-2a`, `SERV-2b` | | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c` + 命中模型规则 | @@ -60,6 +62,20 @@ confidence: high - 验收：每个公开格式都有 encoder smoke；移除格式只改 canonical owner，协议和校验测试 同步反映。 ^[PR #4718] +## SERV-1c — media decoder 参数只在真实入口可达时对外声明 + +- 触发：替换 video/image reference decoder、接入 loader registry、新增 backend/fps/ + frame-selection 参数或改变坏媒体错误。 +- 强制：只有请求/配置入口能把值传到 loader 时才声明参数可配；否则在最近 + loader 调用处固定实际值。frame sampling 只消费真正传入的 target metadata；decoder + 返回零帧必须继续映射为 `InvalidInputReferenceError`，不能因共享 loader 只记 warning + 就返回空 `VideoFrames`。 +- 禁止：不得留下没有 caller 传入的 backend/fps 形参或死分支；不得因 registry 存在 + 多 backend 就宣称 vLLM-Omni 已支持运行时选择；不得丢失替换前的非空帧 fail-safe。 +- 验收：同一合成视频分别断言 first-N 与 last-N 顺序、数量和 fps；覆盖非法 + `keep`、非正 `max_frames`、无效字节与 loader 零帧，且真实 Qwen multimodal/video + 请求仍通过公开入口。 ^[PR #5085] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 67e8fc59b..6a9d2ad3c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 78c144f3`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fbea7886`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 1c4aca7f \ - --to 78c144f3 \ + --from 78c144f3 \ + --to fbea7886 \ --repo \ --mode report-only ``` From 877238c2ebfa48f192ba3699728bf2ccad620c04 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:47:51 +0800 Subject: [PATCH 015/465] knowledge(vllm-omni): sync c9f2e5ad (PR #5775) Upstream-Commit: c9f2e5ad94032b8f2493ca829560efb607e26b79 Upstream-PR: #5775 Knowledge-Delta: baseline-pins-and-bagel-mrope-cfg-position-layout --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/models/bagel/_index.md | 3 +- .../vllm-omni/models/bagel/architecture.md | 7 +++-- .../repos/vllm-omni/models/bagel/rules.md | 31 +++++++++++++++++++ knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../repos/vllm-omni/models/lance/_index.md | 4 ++- .../vllm-omni/models/lance/architecture.md | 10 ++++-- 14 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/bagel/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 49c9e337d..03a8487f5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 78c144f3a8f1e4fb3e9d9e0c38bc0a0e635c7c98 + previous_audited_sha: fbea7886682e332c0450c5e4d0c75b330b661166 audited_ref: main - audited_sha: fbea7886682e332c0450c5e4d0c75b330b661166 + audited_sha: c9f2e5ad94032b8f2493ca829560efb607e26b79 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e5a9361b6..379c90c57 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - fbea7886682e332c0450c5e4d0c75b330b661166`. The canonical machine baseline + c9f2e5ad94032b8f2493ca829560efb607e26b79`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 60732324f..a9271eb46 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fbea7886` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c9f2e5ad` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 57f8e9981..a50df76e3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fbea7886` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c9f2e5ad` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 34cdeec95..9e8433685 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fbea7886` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c9f2e5ad` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 743bc5d84..72fdaee72 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fbea7886` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c9f2e5ad` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bfc79511f..0cc582e20 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fbea7886` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c9f2e5ad` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4c3ca23bd..9e976ac07 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fbea7886` 验证存在 +- 源码校验：以上路径均已在 `main @ c9f2e5ad` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/bagel/_index.md b/knowledge/repos/vllm-omni/models/bagel/_index.md index 52cf19bf9..37b51c28e 100644 --- a/knowledge/repos/vllm-omni/models/bagel/_index.md +++ b/knowledge/repos/vllm-omni/models/bagel/_index.md @@ -1,7 +1,7 @@ --- title: "BAGEL（统一模型多形态部署参照）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] sources: [vllm_omni/model_executor/models/bagel/, vllm_omni/diffusion/models/bagel/, vllm_omni/deploy/bagel.yaml] @@ -44,6 +44,7 @@ sources: [vllm_omni/model_executor/models/bagel/, vllm_omni/diffusion/models/bag | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | KV 桥接、3 路 CFG、MoT、变体拓扑 | [architecture](architecture.md) | AR→DiT 数据流与 reviewer 陷阱 | +| CFG 分支 position ID 打包 | [rules](rules.md) | 1-D RoPE 与 Lance mRoPE 的序列轴合同 | ## 配置与 checkpoint 差异 diff --git a/knowledge/repos/vllm-omni/models/bagel/architecture.md b/knowledge/repos/vllm-omni/models/bagel/architecture.md index a98837cb3..65026057c 100644 --- a/knowledge/repos/vllm-omni/models/bagel/architecture.md +++ b/knowledge/repos/vllm-omni/models/bagel/architecture.md @@ -1,10 +1,10 @@ --- title: "BAGEL 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: architecture tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/model_executor/models/bagel/bagel.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/model_executor/stage_input_processors/bagel.py] +sources: [vllm_omni/model_executor/models/bagel/bagel.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/model_executor/stage_input_processors/bagel.py] --- # BAGEL 架构 @@ -65,6 +65,9 @@ sources: [vllm_omni/model_executor/models/bagel/bagel.py, vllm_omni/diffusion/mo 4. DiT stage 用 `NaiveCache.from_object` 重建主 KV,`collect_cfg_kv_caches` 收伴随 KV;`Bagel.generate_image` 去噪;可选轨迹记录 （`return_trajectory_latents`,RL 用）→ PIL Image。 +- 顺序 CFG 把各分支 query/cache 合并为一次 LLM forward 时，position ID 的拼接 + 轴取决于表示秩：BAGEL 标量 RoPE 的 `(S,)` 沿 dim 0 拼；Lance mRoPE 的 + `(3,S)` 沿 dim 1 拼。可执行验收见 [BAGEL 规则 BAGEL-1](rules.md)。 - RNG 陷阱：`_regen_init_noise_on_device` 按请求在 CUDA 上重播 init noise （对齐 Lance 噪声流）;`forward` 单 prompt（多了只取第一个并告警）;分辨率超 `max_latent_size × latent_downsample` 直接 raise。 diff --git a/knowledge/repos/vllm-omni/models/bagel/rules.md b/knowledge/repos/vllm-omni/models/bagel/rules.md new file mode 100644 index 000000000..77b8061b5 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/bagel/rules.md @@ -0,0 +1,31 @@ +--- +title: "BAGEL 实现规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5775", vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/lance/lance_transformer.py] +confidence: high +--- + +# BAGEL 实现规则 + +## BAGEL-1：CFG position ID 沿序列轴合并 + +**适用范围**：`Bagel.forward` 将顺序 CFG 的 gen/cfg_text/cfg_img 分支合并为一次 +LLM forward；也覆盖继承 `Bagel.generate_image` 和该 forward 的 Lance。 + +**合同**：每分支的普通 RoPE position ID 形状为 `(S,)`，沿 dim 0 合并为 +`(sum S,)`；每分支的 multimodal RoPE position ID 形状为 `(3,S)`，必须沿 dim 1 +合并为 `(3,sum S)`。三条 t/h/w 是模态轴，不能沿 dim 0 拼成 `(3N,S)`；CFG 的 +batch/branch 关系由重复 query sequence、`packed_seqlens` 和 merged cache 单独表达。 + +**验收**： + +- 构造至少两个 CFG 分支，分别断言 1-D 输入得到 `(sum S,)`、2-D 输入得到 + `(3,sum S)`，且每个分支在序列轴上的值和顺序保持不变。 +- 将结果送到对应 rotary/mRoPE 下游，断言 position ID 的序列长度与重复后的 query + token 总数一致，模态轴仍为 3；不能只检查 `torch.cat` 本身不报错。 +- 保留 BAGEL 1-D 路径的回归覆盖，并为 Lance `(3,S)` 路径补 GPU E2E。引入该合同的 + PR #5775（`c9f2e5ad`）只有 shape tracing、ruff 和 reviewer approval，作者因无 + CUDA GPU 未执行 Lance 端到端，因此不能把该提交视为 GPU 精度/功能已验证。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6a9d2ad3c..49aa95aaa 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fbea7886`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c9f2e5ad`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 78c144f3 \ - --to fbea7886 \ + --from fbea7886 \ + --to c9f2e5ad \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/lance/_index.md b/knowledge/repos/vllm-omni/models/lance/_index.md index e28fbea8f..910b59465 100644 --- a/knowledge/repos/vllm-omni/models/lance/_index.md +++ b/knowledge/repos/vllm-omni/models/lance/_index.md @@ -1,7 +1,7 @@ --- title: "Lance（BAGEL 谱系统一模型,单 DIFFUSION stage 全模态）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] sources: [vllm_omni/diffusion/models/lance/, vllm_omni/model_executor/models/lance/pipeline.py, vllm_omni/deploy/lance.yaml] @@ -39,6 +39,8 @@ sources: [vllm_omni/diffusion/models/lance/, vllm_omni/model_executor/models/lan - 依赖共享模块：bagel 家族的 `Bagel.generate_image` 去噪 （`scheduler=None`,timestep-shift flow）、 [Diffusion 组件](../../components/diffusion/_index.md)。 +- Lance 的 `(3,S)` mRoPE position ID 也经过 Bagel 的 CFG 分支合并；修改该路径时 + 必须满足 [BAGEL-1](../bagel/rules.md)，不能把三条模态轴拼成 `(3N,S)`。 ## 目录内容 diff --git a/knowledge/repos/vllm-omni/models/lance/architecture.md b/knowledge/repos/vllm-omni/models/lance/architecture.md index f2718cfd6..fd8390abd 100644 --- a/knowledge/repos/vllm-omni/models/lance/architecture.md +++ b/knowledge/repos/vllm-omni/models/lance/architecture.md @@ -1,7 +1,7 @@ --- title: "Lance 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: architecture tags: [vllm-omni, models, diffusion] sources: [vllm_omni/diffusion/models/lance/pipeline_lance.py, vllm_omni/diffusion/models/lance/lance_transformer.py, vllm_omni/diffusion/models/lance/wan_vae.py, vllm_omni/diffusion/models/lance/prompts.py] @@ -52,7 +52,9 @@ sources: [vllm_omni/diffusion/models/lance/pipeline_lance.py, vllm_omni/diffusio Lance 对齐;`_extract_user_instruction` 从渲染后模板抽回原始用户文本。 3. **CFG 是 BAGEL 式双 KV 上下文**（`cfg_text_context` 维护负向/无条件 prompt 的第二套 past-KV,一起交给 `generate_image`）——**不是** - CFGParallelMixin,无 rank 切分。 + CFGParallelMixin,无 rank 切分。各分支 mRoPE position ID 是 `(3,S)`；共享 + Bagel forward 合并分支时沿 dim 1 扩展序列，保留三条模态轴，详见 + [BAGEL-1](../bagel/rules.md)。 4. 视频路:3D latent + `LanceWanVAE.decode_video`。 - **bring-up 状态（pin 上如实记录,docstring 与代码有陈旧差）**：docstring 称 t2i 端到端验证（B300,~6 s@1024²）、x2t 立即 EOS（缺 mRoPE 端到端位置 @@ -68,6 +70,10 @@ pin 上仅识别到一个 lance 测试路径 示例 `examples/{offline_inference,online_serving}/lance/`（含 gradio）。 本次调查未发现精度基线或性能 gate——相关结论需另行实测。 +`c9f2e5ad` 修复的 CFG/mRoPE 拼接仅以 shape tracing、ruff 和一次 reviewer approval +验收；作者没有 CUDA GPU，未跑 Lance 端到端。后续修改需同时覆盖 1-D/2-D position +ID 与 rotary 下游 shape，并把 Lance GPU E2E 视为仍待补的验证边界。 + - 已知未决：`get_lance_pre_process_func` 未注册是否有意; `Qwen2MoTConfig/ForCausalLM` 是 lance_transformer 定义还是从 bagel 转 出口;x2t 路由的文本如何经 `final_output_type="image"` 的 stage 交付。 From 8d26abcfd731492b20ba77e9f321fbd4c4433e4e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:51:50 +0800 Subject: [PATCH 016/465] knowledge(vllm-omni): sync bfaaa3c2 (PR #5780) Upstream-Commit: bfaaa3c2c3fb41f32f3015dc20958cc770eb5f38 Upstream-PR: #5780 Knowledge-Delta: baseline-pins-and-pytest-package-collection --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 15 ++++++++++----- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 22 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 03a8487f5..f323ed1d2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fbea7886682e332c0450c5e4d0c75b330b661166 + previous_audited_sha: c9f2e5ad94032b8f2493ca829560efb607e26b79 audited_ref: main - audited_sha: c9f2e5ad94032b8f2493ca829560efb607e26b79 + audited_sha: bfaaa3c2c3fb41f32f3015dc20958cc770eb5f38 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 379c90c57..e17fb06cc 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - c9f2e5ad94032b8f2493ca829560efb607e26b79`. The canonical machine baseline + bfaaa3c2c3fb41f32f3015dc20958cc770eb5f38`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index a9104d47e..7c97f4ea9 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni CI 规则" created: 2026-08-23 -updated: 2026-08-23 +updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5524", "PR #5543", "PR #5670", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] +sources: ["PR #3422", "PR #5074", "PR #5524", "PR #5543", "PR #5670", "PR #5780", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] confidence: high --- @@ -25,11 +25,16 @@ confidence: high - 触发：增加模型、硬件路径、nightly/ready/merge lane，或升级运行时/镜像。 - 强制：资源数、pytest marker、pipeline 路由和共享 stage registry 同步；必需依赖、模型初始化 - 或 collection 失败必须让 job 失败，并证明目标测试数大于零且原失败在对应硬件消失。 + 或 collection 失败必须让 job 失败，并证明目标测试数大于零且原失败在对应硬件消失。默认 + `prepend` import mode 下，whole-tree collection 遇到同名叶子目录时，所有中间父目录必须用 + `__init__.py` 保持规范限定包名，不能让不同子树都退化成同一个顶层包。 - 禁止：初始化失败后 skip-green；用 CPU 静态检查代替 CUDA/NPU 执行；只改模型目录而漏掉 共享 registry/路由；把临时硬件硬编码留到合并。 -- 验收：当前 head 的目标 lane 记录硬件、依赖版本、collection 数和执行结果；注入缺依赖、坏 - registry 或错误 marker 时 lane 必须红。 ^[PR #3422] ^[PR #5524] ^[PR #5543] +- 验收：当前 head 按真实 lane 的 whole-tree/marker scope 记录硬件、依赖版本、collection 数和 + 执行结果，而非只收目标测试文件；同名叶子包必须解析为不同规范名（如 + `tests.e2e.accuracy.minimax_h3` 与 `tests.diffusion.models.minimax_h3`）。注入缺 + `__init__.py`、缺依赖、坏 registry 或错误 marker 时 lane 必须红。 ^[PR #3422] + ^[PR #5524] ^[PR #5543] ^[PR #5780] ^[PR #5872] ^[PR #6048] ^[PR #6096] ^[PR #6102] ## OMNI-CI-1b — 回归 fence 必须锚定可观察合同且非空转 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a9271eb46..46306e16b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c9f2e5ad` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bfaaa3c2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a50df76e3..e18020811 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c9f2e5ad` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bfaaa3c2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9e8433685..90e65e695 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c9f2e5ad` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bfaaa3c2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 72fdaee72..cfe0e8d94 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c9f2e5ad` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bfaaa3c2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0cc582e20..9aaadfede 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c9f2e5ad` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bfaaa3c2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9e976ac07..fa8524296 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c9f2e5ad` 验证存在 +- 源码校验：以上路径均已在 `main @ bfaaa3c2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 49aa95aaa..4a41dc528 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c9f2e5ad`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bfaaa3c2`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from fbea7886 \ - --to c9f2e5ad \ + --from c9f2e5ad \ + --to bfaaa3c2 \ --repo \ --mode report-only ``` From c825611a73aea965777d43f8329c275738676ce9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 12:57:29 +0800 Subject: [PATCH 017/465] knowledge(vllm-omni): sync f24a6165 (PR #5310) Upstream-Commit: f24a6165ff1612b912962e7e169ebfdd1d2a080a Upstream-PR: #5310 Knowledge-Delta: baseline-pins-and-prefix-cache-passthrough-tests --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 8 +++--- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 6 +++-- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 27 ++++++++++++++++--- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 12 files changed, 45 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f323ed1d2..3ea6f9e25 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c9f2e5ad94032b8f2493ca829560efb607e26b79 + previous_audited_sha: bfaaa3c2c3fb41f32f3015dc20958cc770eb5f38 audited_ref: main - audited_sha: bfaaa3c2c3fb41f32f3015dc20958cc770eb5f38 + audited_sha: f24a6165ff1612b912962e7e169ebfdd1d2a080a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e17fb06cc..6f7b49f3f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - bfaaa3c2c3fb41f32f3015dc20958cc770eb5f38`. The canonical machine baseline + f24a6165ff1612b912962e7e169ebfdd1d2a080a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 7c97f4ea9..17264eb69 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5524", "PR #5543", "PR #5670", "PR #5780", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] +sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5524", "PR #5543", "PR #5670", "PR #5780", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] confidence: high --- @@ -29,12 +29,14 @@ confidence: high `prepend` import mode 下，whole-tree collection 遇到同名叶子目录时，所有中间父目录必须用 `__init__.py` 保持规范限定包名，不能让不同子树都退化成同一个顶层包。 - 禁止：初始化失败后 skip-green；用 CPU 静态检查代替 CUDA/NPU 执行；只改模型目录而漏掉 - 共享 registry/路由；把临时硬件硬编码留到合并。 + 共享 registry/路由；把临时硬件硬编码留到合并。marker 名不等于实际硬件：某目录只有 + `core_model and cpu` lane 时，GPU-dependent case 也应使用该 lane selector，再用 CUDA + availability skipif 作硬件门；标成无人选择的 `cuda` 会得到零 collection。 - 验收：当前 head 按真实 lane 的 whole-tree/marker scope 记录硬件、依赖版本、collection 数和 执行结果，而非只收目标测试文件；同名叶子包必须解析为不同规范名（如 `tests.e2e.accuracy.minimax_h3` 与 `tests.diffusion.models.minimax_h3`）。注入缺 `__init__.py`、缺依赖、坏 registry 或错误 marker 时 lane 必须红。 ^[PR #3422] - ^[PR #5524] ^[PR #5543] ^[PR #5780] + ^[PR #5310] ^[PR #5524] ^[PR #5543] ^[PR #5780] ^[PR #5872] ^[PR #6048] ^[PR #6096] ^[PR #6102] ## OMNI-CI-1b — 回归 fence 必须锚定可观察合同且非空转 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 46306e16b..ee0f876dc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bfaaa3c2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f24a6165` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e18020811..5a0bc9490 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bfaaa3c2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f24a6165` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 90e65e695..1cb59557a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bfaaa3c2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f24a6165` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cfe0e8d94..f4f6c7d90 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bfaaa3c2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f24a6165` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 7821ecbe7..64a962cf0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5074", "PR #5610", "PR #5777", "PR #5792", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5074", "PR #5310", "PR #5610", "PR #5777", "PR #5792", "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -146,7 +146,9 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：输出按请求索引与输入一一对应；无法安全逐请求处理时把并发上限显式收紧为 1。 - 禁止：只消费 `runtime_info[0]`，或把单元素 waveform/metadata 广播给整个 batch。 - 验收：至少两个不同输入的同批测试，分别断言 bridge、输出和错误归属；不能重复相同 - prompt 让串线不可见。 ^[PR #3642] + prompt 让串线不可见。partial downstream subset 必须保留原 `req_id_to_index`：跳过 + 中间请求后，后续请求的 tensor slice 和 list-valued payload 仍取原 batch index，不能 + 压缩到 downstream position。 ^[PR #3642] ^[PR #5310] ### EXEC-1c — 请求随机状态跨 batching 和 yield 保持请求所有权 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9aaadfede..46cc8e53e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bfaaa3c2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f24a6165` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 2a7cf9aa6..a18c33d13 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: rule tags: [vllm-omni, components, scheduler] -sources: ["vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5461"] +sources: ["vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461"] --- # Scheduler 规则 @@ -20,7 +20,7 @@ PR 描述先命中下表，再打开对应规则组和首批源码；changed fil | PR 描述信号 | 规则组 | 第一批源码 | |---|---|---| -| prefix cache、truncated prefill、mm key、deferred payload | SCHED-1a | `core/prefix_cache.py::OmniTensorPrefixCache.maybe_init_missing_mm_cache_keys`；`worker/gpu_model_runner.py::initialize_metadata_builders`；`worker/gpu_ar_model_runner.py::_deferred_prefix_cache_mm_keys` | +| prefix cache、truncated prefill、mm key、deferred/passthrough payload | SCHED-1a/1b | `core/prefix_cache.py::OmniTensorPrefixCache`；`utils/mm_outputs.py::{build_mm_cpu,to_payload_element}`；`worker/gpu_ar_model_runner.py::_deferred_prefix_cache_mm_keys` | | `max_num_batched_tokens`、prefill throttle、低预算 GPU hang | SCHED-2a | `core/sched/omni_ar_scheduler.py::OmniARScheduler.schedule`；`core/sched/omni_generation_scheduler.py::OmniGenerationScheduler.schedule`；触发它的 deploy/test 配置 | | vLLM bump、scheduler rebase、KV connector stats | SCHED-3a | 两个 scheduler 的 `schedule` / `update_from_output` 与 live upstream `vllm/v1/core/sched/` | | side-stream D2H、pinned host tensor、源 buffer 复用 | SCHED-4a/4b | `worker/gpu_ar_model_runner.py::_copy_tensor_payload_to_cpu`、`_get_or_create_omni_payload_copy_stream`；`core/prefix_cache.py` 的 async copy 路径 | @@ -101,6 +101,24 @@ modules=[worker_runner, model_executor]，status=active，run_count=54，2026-06 机制，机制变了要复查；绕过 cap 的缓存住在主机内存（本例每 key ~2.6 GiB）——TB 级内存的 CI 主机 可承受，小内存主机要标注。^[SK-fix-talker-truncated-prefill-prefix-cache-key-cap] +## SCHED-1b — prefix-cache passthrough 必须保留逐请求边界与载荷语义 + +- 触发：修改 `build_mm_cpu`、`to_payload_element`、prefix-cache merge/deferred payload， + 或 runner 对 combined multimodal output 的拆包。 +- 强制：token-aligned tensor（首维等于本步 scheduled token 总数）按每个请求的 + `start:end` 切片；非 token-aligned tensor 给每个请求完整独立 clone；prefix-cache + merge 中的 list（如 `codes.ref`）保持整表并逐请求 clone，直到 runner 用**原 batch + index** `_unwrap_lists`。D2H 后每个嵌套 tensor leaf 必须在 CPU、detached、contiguous， + 并保持 dtype/shape/value；dict/list 与非 tensor leaf 的结构和值不变。 +- 生命周期：abort/discard 必须删除 staged chunks，之后的 commit 是 no-op；同 req-id + 重用从空状态开始，不能混入旧 chunk。 +- 禁止：把所有 passthrough tensor 一律按 token 切；在 merge 阶段提前按 downstream + subset index 拆 list；用 float staging buffer 改写 int metadata；把本规则解释为后续 W2 + 优化已经实现——`f24a6165` 只增加当前行为的 characterization tests。 +- 验收：mixed hit/miss + 不等 scheduled length 同时覆盖 1-D/2-D token-aligned、多个 + 非对齐 shape、per-request tensor list 和 int32/int8 metadata；修改一请求结果不影响 + 其他请求或源值。另测 discard→late commit、discard→req-id reuse。^[PR #5310] + ## SCHED-2a — 极小 max_num_batched_tokens 与并发 prefill 会挂死 GPU skill 元数据：`gpu-hang-low-max-num-batched-tokens`， @@ -143,10 +161,13 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 的 `slot_mapping`/hidden/mm tensor。 - 强制：源 tensor 在 copy event 完成前保持有效且不可被重写；使用显式 stream ordering、 retain/`record_stream` 或消费屏障。drain/early-return 也必须完成或转移生命周期责任。 + 单 pending-write 流水线中 event 未 ready 时 drain 必须不触碰 cache；ready 后恰好消费 + 一次；调度第 N+1 次 write 前必须先消费第 N 次，不能覆盖 pending state。 - 禁止：仅同步目标 CPU tensor，却允许下一 step 复用源 GPU buffer；这会得到合法 shape 但错误行内容。 - 验收：连续两 step 写入不同 sentinel，在人为延迟 side stream 下证明第一轮 CPU 结果 - 不被第二轮覆盖。 ^[PR #4106] + 不被第二轮覆盖；强制 event query false→true，断言 0→1→0 次 drain，并验证未显式 + drain 时下一次 schedule 仍先落盘上一轮。 ^[PR #4106] ^[PR #5310] ## SCHED-4b — pinned CPU 分配必须先判断 CUDA 能力 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fa8524296..37d576a46 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bfaaa3c2` 验证存在 +- 源码校验：以上路径均已在 `main @ f24a6165` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4a41dc528..d0847357c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bfaaa3c2`（2026-08-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f24a6165`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from c9f2e5ad \ - --to bfaaa3c2 \ + --from bfaaa3c2 \ + --to f24a6165 \ --repo \ --mode report-only ``` From 08af701251b37d635a4c3f38d5a046c99f5f6335 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 13:05:18 +0800 Subject: [PATCH 018/465] knowledge(vllm-omni): sync e788ef6e (PR #4771) Upstream-Commit: e788ef6e76a93e8fe73c3072064f985ea0b50889 Upstream-PR: #4771 Knowledge-Delta: baseline-inventories-and-personaplex-model-owner --- adapters/vllm_omni/release_baseline.yaml | 16 ++--- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/_index.md | 5 +- knowledge/repos/vllm-omni/models/catalog.md | 22 +++--- .../vllm-omni/models/personaplex/_index.md | 38 ++++++++++ .../models/personaplex/architecture.md | 52 ++++++++++++++ .../vllm-omni/models/personaplex/rules.md | 70 +++++++++++++++++++ 13 files changed, 191 insertions(+), 26 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/personaplex/_index.md create mode 100644 knowledge/repos/vllm-omni/models/personaplex/architecture.md create mode 100644 knowledge/repos/vllm-omni/models/personaplex/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3ea6f9e25..723a7fdac 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bfaaa3c2c3fb41f32f3015dc20958cc770eb5f38 + previous_audited_sha: f24a6165ff1612b912962e7e169ebfdd1d2a080a audited_ref: main - audited_sha: f24a6165ff1612b912962e7e169ebfdd1d2a080a + audited_sha: e788ef6e76a93e8fe73c3072064f985ea0b50889 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 77 - sha256: 10e7850b0c7804f23ddffeb7cf5f2ad83c616d3977ec3b67eabab5c121564a49 + count: 79 + sha256: 700ab6a4918de2d796604d1e6da5f619731a53bb09b8c56416f253e820ffdf3a diffusion: count: 58 sha256: 7dc6a5a373ca33a31a44961c1422008327cede3b364577fa095fd37c05ea2085 pipelines: - count: 51 - sha256: 8f567368a2a958fee2f89e4bce3ec20ec2e7a420bbdbab2fef9d65fb604e3ae8 + count: 52 + sha256: 54a31adef93bc33e2009252699075930ee05ffb38605a52ba9370a21c542b320 deploy_yamls: - count: 79 - sha256: fb75633eeeaadc4f421fd4d2ec62b03b04d9a9f48e959297d88ae51c20d51bc5 + count: 80 + sha256: 9fadef14f3ec845ec7a7060df69250badd9a7f002bf5962db229d32c38185397 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6f7b49f3f..73828e71b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - f24a6165ff1612b912962e7e169ebfdd1d2a080a`. The canonical machine baseline + e788ef6e76a93e8fe73c3072064f985ea0b50889`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ee0f876dc..5ade8ab3a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f24a6165` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e788ef6e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5a0bc9490..9ba838a32 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f24a6165` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e788ef6e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1cb59557a..2b826e5f1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f24a6165` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e788ef6e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f4f6c7d90..62412689b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f24a6165` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e788ef6e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 46cc8e53e..313fafaca 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f24a6165` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e788ef6e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 37d576a46..040df59d9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f24a6165` 验证存在 +- 源码校验：以上路径均已在 `main @ e788ef6e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/_index.md b/knowledge/repos/vllm-omni/models/_index.md index ae2e5f19a..20b73b415 100644 --- a/knowledge/repos/vllm-omni/models/_index.md +++ b/knowledge/repos/vllm-omni/models/_index.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni 模型 owner" created: 2026-07-10 -updated: 2026-07-31 +updated: 2026-09-02 type: index tags: [vllm-omni, models] sources: [] @@ -27,5 +27,8 @@ Get-ChildItem knowledge/repos/vllm-omni/models -Directory | 4. 目录只有 `_index.md` 时，把它当源码落脚页；有 `rules.md` 时优先读规则； 只有需要解释完整拓扑时才读 `architecture.md`。 +PersonaPlex 的 staged 与 unified full-duplex 双入口、Mimi 状态和 lockstep 合同见 +[PersonaPlex owner](personaplex/_index.md)。 + 选择一个模型 owner 后停止枚举其他模型。跨模型共用的不变量应进入 [components](../components/_index.md)，不要复制到多个模型目录。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d0847357c..64adef6bd 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f24a6165`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e788ef6e`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -35,23 +35,25 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| -| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 77 个架构名 / 27 个模型族目录 | +| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 79 个架构名 / 28 个模型族目录 | | Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 58 条 pipeline / 38 个模型族目录 | -| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 51 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 79 份 | +| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 52 个 key | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 80 份 | 对比上一审计快照（`807db6ef`）：AR 架构 72→77，diffusion pipeline 61→58， OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusion 新增 `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 +本次相对 `f24a6165` 新增 PersonaPlex 的两个 AR architecture、`personaplex` pipeline key +和 `personaplex.yaml` deploy；diffusion inventory 不变。 -## AR/omni 模型族（27） +## AR/omni 模型族（28） aura_omni、audex、bagel、cosyvoice3、covo_audio、dynin_omni、fish_speech、glm_image、 glm_tts、higgs_audio_v2、higgs_audio_v3、hunyuan_image3、indextts2、 mammoth_moda2、mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、moss_tts、 -moss_tts_nano、omnivoice、qwen2_5_omni、qwen3_omni、qwen3_tts、step_audio2、 +moss_tts_nano、omnivoice、personaplex、qwen2_5_omni、qwen3_omni、qwen3_tts、step_audio2、 voxcpm2、voxtral_tts ## Diffusion 模型族（38） @@ -63,7 +65,7 @@ lingbot_video、longcat_image、ltx2、magi_human、ming_flash_omni、nextstep_1 omnivoice、ovis_image、qwen_image、sd3、sdxl、sensenova_u1、soulx_singer、 stable_audio、wan2_2、z_image、minimax_h3 -## OMNI_PIPELINES key（51） +## OMNI_PIPELINES key（52） Gr00tN1d7（注意:唯一 CamelCase key）、aura_omni、audex_s2s、audex_thinker_only、 audex_tta、audex_tts、bagel、bagel_single_stage、 @@ -73,7 +75,7 @@ hunyuan_image3_dit、hunyuan_image_3_moe、hunyuan_video_15、indextts2、lance mammoth_moda2、mammoth_moda2_ar、mimo_audio、ming_flash_omni、 ming_flash_omni_image、ming_flash_omni_thinker_only、ming_flash_omni_tts、 ming_tts、ming_tts_moe、minicpmo_4_5、moss_tts_delay、moss_tts_local、 -moss_tts_nano、moss_tts_realtime、omnivoice、qwen2_5_omni、 +moss_tts_nano、moss_tts_realtime、omnivoice、personaplex、qwen2_5_omni、 qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_tts、 soulxsinger_svc、soulxsinger_svs、step_audio_2、step_audio_2_asr、voxcpm2、 voxtral_tts、wan2_2_ti2v、nemotron_labs_audex @@ -87,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from bfaaa3c2 \ - --to f24a6165 \ + --from f24a6165 \ + --to e788ef6e \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/personaplex/_index.md b/knowledge/repos/vllm-omni/models/personaplex/_index.md new file mode 100644 index 000000000..35003dc83 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/personaplex/_index.md @@ -0,0 +1,38 @@ +--- +title: "PersonaPlex" +created: 2026-09-02 +updated: 2026-09-02 +type: index +tags: [vllm-omni, models, model-executor] +sources: ["PR #4771", vllm_omni/model_executor/models/personaplex/, vllm_omni/experimental/fullduplex/personaplex/, vllm_omni/deploy/personaplex.yaml] +confidence: high +--- + +# PersonaPlex + +## 名称、注册与范围 + +- 正式模型 `nvidia/personaplex-7b-v1`，是 Moshi-based full-duplex S2S；目标 pin + `e788ef6e` 的实现标为 experimental。 +- AR registry 新增 `PersonaPlexTalkerForConditionalGeneration` 和 + `PersonaPlexCode2Wav`，都位于 `model_executor/models/personaplex/`；pipeline registry + key 是 `personaplex`，默认 deploy 是 `personaplex.yaml`。 +- checkpoint 的 `config.json` 为空；`engine/arg_utils.py::_ARCH_TO_MODEL_TYPE` 必须从上述 + architecture 回填 `model_type=personaplex`，再注册本地 `PersonaPlexConfig`。 +- 两个入口必须区分：标准 omni engine 的 talker→code2wav staged pipeline，以及 + `/v1/realtime?duplex=1` unified engine path；`personaplex/serving/` 的 standalone server + 只是兼容面，不能证明 unified path 可用。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| stage、frame、codec/session 状态和 serving 边界 | [architecture](architecture.md) | +| frame accounting、de-delay、lease、greedy/config 门禁 | [rules](rules.md) | + +## 什么时候查这里 + +- 审查 PersonaPlex registry/config、Mimi codec、async-chunk handoff、duplex admission、 + session cleanup 或 WebSocket reconnect。 +- 共享 full-duplex 生命周期转到 [Serving rules](../../components/serving/rules.md)，共享 + runner hook/config projection 转到 [Model Executor rules](../../components/model-executor/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/personaplex/architecture.md b/knowledge/repos/vllm-omni/models/personaplex/architecture.md new file mode 100644 index 000000000..d04634fd7 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/personaplex/architecture.md @@ -0,0 +1,52 @@ +--- +title: "PersonaPlex 架构" +created: 2026-09-02 +updated: 2026-09-02 +type: architecture +tags: [vllm-omni, models, model-executor] +sources: ["PR #4771", vllm_omni/experimental/fullduplex/personaplex/DESIGN.md, vllm_omni/model_executor/models/personaplex/pipeline.py, vllm_omni/deploy/personaplex.yaml] +confidence: high +--- + +# PersonaPlex 架构 + +以下只描述 `main @ e788ef6e`；入口速览见 [index](_index.md)，可执行门禁见 +[rules](rules.md)。 + +## 模型与 staged pipeline + +- Stage 0 `PersonaPlexTalkerForConditionalGeneration`（LLM_AR）包含 Helium temporal、 + depformer 和 input embeddings；每个 80 ms frame 产生 inner-monologue text 与 audio + codebooks。runner 的 model-owned `post_sample_talker_mtp` hook 用本步 sampled text 和 + temporal hidden state 生成同 frame codec codes。 +- Stage 1 `PersonaPlexCode2Wav`（LLM_GENERATION）用 Mimi 把 8 个 active codebooks 解为 + 24 kHz mono PCM；两 stage 通过 SharedMemoryConnector async-chunk streaming 连接。 + Stage 1 默认 eager，因为 decoder 的 host→device copy 不能安全 capture。 +- 帧合同固定为 24 kHz、1920 samples、80 ms（12.5 Hz）；17 行 token column 是 text 1、 + agent audio 8、user audio 8。标准 staged 路径最终输出 audio。 + +## Unified full-duplex path + +```text +/v1/realtime?duplex=1 + → session actor / correlated RPC / DuplexControlPlane + → resumable Stage 0 → Talker → streaming Code2Wav + → data-plane projector → audio delta + transcript delta +``` + +- serving adapter 只拥有 PCM reservation、client validation、capability 和 output cursor； + runtime extension 只把一帧 append 映射为 scheduler prompt/metadata/token budget。真实 Mimi + encode、voice/persona prefill、delayed frame 和 KV 由 model-owned Stage 0 runtime 管理。 +- 每个 `(session_id, incarnation)` 有独立 Stage 0 Mimi encoder；每个稳定 Stage 1 request id + 有独立 streaming decoder。finish/abort/close 只释放本 session/request 的状态。 +- `duplex_session.max_sessions: 2` 是 engine admission 与两 stage codec pool 的唯一容量源， + 通过 `duplex_max_sessions` 投影到所有 stage；connector extra 不另设容量。 +- capability 在该 pin 明确 `supports_barge_in=false`、不支持 session resume、audio truncate + 或 rollback。模型原生同时听说不等于可破坏性回滚的 barge-in 合同。 + +## 验证边界 + +target tests 覆盖 config/pipeline、Stage 0/1 state isolation、frame accounting、mixed sessions、 +capacity rejection/reuse、cleanup、data-plane delta、standalone lease 和 unified runtime；PR +记录一次 collaborator approval。模型仍是 experimental，且 greedy-only；不要把 PR body 的 +历史性能数字或 standalone server smoke 当作当前环境的保证。 diff --git a/knowledge/repos/vllm-omni/models/personaplex/rules.md b/knowledge/repos/vllm-omni/models/personaplex/rules.md new file mode 100644 index 000000000..6e27fa2c0 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/personaplex/rules.md @@ -0,0 +1,70 @@ +--- +title: "PersonaPlex 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, model-executor] +sources: ["PR #4771", vllm_omni/experimental/fullduplex/personaplex/DESIGN.md, vllm_omni/model_executor/stage_input_processors/personaplex.py, tests/e2e/features/fullduplex/] +confidence: high +--- + +# PersonaPlex 规则 + +只有 `PPLEX-数字字母` 是可审计规则 ID。 + +## PPLEX-1a — frame 与 de-delay 边界必须跨 chunk 连续 + +- 触发：PCM append、Mimi encode/decode、talker→code2wav processor 或 chunk flush。 +- 强制：输入严格是 24 kHz mono float PCM；每个物理 append unit 是 1920 samples/80 ms， + reservation 失败回滚相同 bytes 且不推进 frame cursor。acoustic de-delay 使用下一 frame + 构造 cb1..7，所以非 final chunk 必须保留最后 raw code frame 给下一 chunk。 +- 禁止：按任意 client packet 直接推进模型；每 chunk 丢弃边界 frame；把空/全零/非有限 + audio 或只含 listen event 视为成功。 +- 验收：覆盖 partial-frame final padding、prepare/submit/commit rollback、重复 operation id、 + 跨两个 chunk 的 codebook 对齐与 frame/audio 时长计数。^[PR #4771] + +## PPLEX-1b — codec 状态和容量以 session 身份隔离 + +- 触发：并发 session、slot reuse、abort/close、Stage 0/1 codec pool 或 admission 配置。 +- 强制：Stage 0 encoder 按完整 `(session_id, incarnation)`、Stage 1 decoder 按稳定 request id + 独占；`duplex_session.max_sessions` 是 admission 与所有 codec pool 的单一容量源。cleanup + 只回收目标身份，replacement session 必须重建 voice/persona、PCM tail、Mimi state 与 + delayed frame。 +- 禁止：在 connector extra 复制容量；共享卷积/KV state；timeout 后工作仍在运行却提前 + 释放 slot。 +- 验收：两 session 交错产生独立非空输出，第三个在上限处拒绝；关闭一条不改变另一条， + replacement 用不同 persona 且无 state leak；异常、timeout、abort 路径无遗留。^[PR #4771] + +## PPLEX-2a — standalone 单会话 lease 必须覆盖不可取消的线程调用 + +- 触发：standalone `DuplexServer`（`batch_size=1`）的 `/api/chat` 或 + `/v1/audio/duplex` disconnect、task cancellation 或快速重连；batched server 另由 + slot manager 的 lock/epoch 隔离并发会话。 +- 强制：`asyncio` task 取消后，`asyncio.to_thread(session.feed)` 的 engine call 仍可能运行； + lease 只能在 out-of-band completion event 证明调用结束后释放，否则继续持有/拒绝新连接。 + `open()`/`feed()` 不能与前 session 的 `step()` 在同一 engine 上并发。 +- 禁止：把 `cancel(); await task` 返回当作 worker thread 已停止；在 finally 直接清 active + lease，使新连接 reset 正被旧线程修改的 KV/cache。 +- 验收：阻塞 feed 后 disconnect，并在其真实完成前 fast reconnect；新会话必须被拒绝或 + 等待，完成后才能 acquire，且没有 CUDA crash 或跨 session state bleed。^[PR #4771] + +## PPLEX-3a — empty checkpoint config 必须从 architecture 恢复本地 config + +- 触发：HF config auto-detection、registry/pipeline 重命名或 staged arch override。 +- 强制：两个 PersonaPlex architecture 都在 `_ARCH_TO_MODEL_TYPE` 映射到 `personaplex`， + 注册本地 `PersonaPlexConfig`；talker config 无 architectures 时补 stage-0 talker arch， + code2wav arch 由 pipeline stage 显式给出。 +- 禁止：假设空 `config.json` 能自动给出 model_type/architecture；要求用户手写 override。 +- 验收：空 config fixture 从 talker/code2wav arch 都解析到 personaplex pipeline，两个 stage + 分别装载正确 class；未知 arch 明确失败。^[PR #4771] + +## PPLEX-3b — sampling capability 必须如实保持 greedy-only + +- 触发：serving/config CLI、runtime sampling params、text head 或 depformer decoding。 +- 强制：该 pin 的 text 与 depformer 都走 greedy；unified Stage 0 固定 temperature 0、top-k 1、 + 每 segment max_tokens 1。只有真实 sampling consumer 和质量验证到位后才暴露 temperature、 + top-k 或 seed。 +- 禁止：保留无法关闭的 `--greedy` 或未被消费的 sampling knobs；把 continuous overlapping + speech 宣称成 `supports_barge_in=true`。 +- 验收：public capability/config 不出现 dead sampling controls，非 greedy 请求明确拒绝或 + 归一化；capability 保持 `supports_barge_in=false`。^[PR #4771] From e50b7b23c239a98dc9660f25e9afb4201ce19637 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 13:10:47 +0800 Subject: [PATCH 019/465] knowledge(vllm-omni): sync 0a7ac4c6 (PR #5713) Upstream-Commit: 0a7ac4c67e3061ef0a9b4a0283e90dd2981ea88c Upstream-PR: #5713 Knowledge-Delta: baseline-pins-and-sleep-mode-fixture-contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 26 ++++++++++++++++--- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules.md | 7 +++-- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 11 files changed, 39 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 723a7fdac..66163797e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f24a6165ff1612b912962e7e169ebfdd1d2a080a + previous_audited_sha: e788ef6e76a93e8fe73c3072064f985ea0b50889 audited_ref: main - audited_sha: e788ef6e76a93e8fe73c3072064f985ea0b50889 + audited_sha: 0a7ac4c67e3061ef0a9b4a0283e90dd2981ea88c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 73828e71b..2c973bf83 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - e788ef6e76a93e8fe73c3072064f985ea0b50889`. The canonical machine baseline + 0a7ac4c67e3061ef0a9b4a0283e90dd2981ea88c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 17264eb69..2c12423d3 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5524", "PR #5543", "PR #5670", "PR #5780", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] +sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] confidence: high --- @@ -20,6 +20,7 @@ confidence: high | regression/guard、route census、middleware、mutation test | `OMNI-CI-1b` | 公开 app/handler → guard test；先证明旧实现会失败 | | pre-commit、SPDX、shellcheck、stability marker | `OMNI-CI-2a` | `.pre-commit-config.yaml`、`.buildkite/**`、`tools/**` | | xdist、共享 worker、下载 fixture、进程池 | `OMNI-CI-2b` | `tests/conftest.py`、`tests/helpers/**`、`tests/model_tests/**` | +| 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | ## OMNI-CI-1a — 硬件 lane 必须真实收集并执行目标路径 @@ -43,12 +44,15 @@ confidence: high - 触发：新增 bug regression、API guard、route census 或 middleware fence。 - 强制：从 assembled app/公开 handler 观察行为，只改变目标变量；必需 state 不仅检查存在， - 还要检查非 `None`。测试须在旧实现或代表性 mutation 上失败，在修复后通过。 + 还要检查非 `None`。测试须在旧实现或代表性 mutation 上失败，在修复后通过。mock engine + 自己抛出的异常只能证明 route 不吞异常；真实 backend 状态与 HTTP mapping 必须另由 live + server 触发生产操作验证，不能把 mock-authored type/message 当成 backend contract。 - 禁止：绑定可搬迁私有符号或单个 router；用恒真类型断言、错误参数的提前失败、宽泛 `>=400`、不同文本或长度冒充目标分支证据。 - 验收：mutation 分别置空 wiring、移除目标 route/method、绕过 inner middleware，fence 均失败； - route census 覆盖应用实际暴露的 HEAD/OPTIONS 和依赖 state key。 ^[PR #3422] ^[PR #5074] - ^[PR #5670] ^[PR #6202] + route census 覆盖应用实际暴露的 HEAD/OPTIONS 和依赖 state key。异常路径分别用 CPU mock + 证明 route propagation，并用 live assembled app 触发真实 backend condition、核对结构化 + status/body 与失败后的 state。 ^[PR #3422] ^[PR #5074] ^[PR #5670] ^[PR #5713] ^[PR #6202] ## OMNI-CI-2a — CI 工具、schema 和 hook 是可复现供应链 @@ -70,3 +74,17 @@ confidence: high 活对象隐式跨进程传递。 - 验收：OOM、普通异常和 BrokenProcessPool 分别验证隔离/重试；online/offline xdist 均通过， 并发 cache miss 只产生一个完整文件。 ^[PR #6208] ^[PR #6339] + +## OMNI-CI-2c — 昂贵 engine fixture 复用必须恢复状态并隔离拓扑 + +- 触发：为减少 checkpoint cold start，把 function-scoped engine/server 提升到 class/module + scope，或合并 sleep/wake 等重模型 case。 +- 强制：不兼容 topology 使用不同 class-scoped fixture，保证同一时刻只有一个 heavy engine； + module 前后各做一次 device cleanup；每个 case 在 `finally` 将共享 engine wake/reset 到可用 + 状态。不可逆的 level-2 sleep case 放在独立 module-scoped server 的最后，结束后只 teardown。 +- 禁止：共享 engine 后依赖测试顺序留下的 sleep/tag/cache 状态；在 module-scoped + `omni_server` 持有 fixture lock 时再创建 function-scoped server，造成锁等待或 GPU 争用； + 为省 cold start 删除 topology、TP 或 post-wake generation 的独立覆盖。 +- 验收：分别覆盖 LLM、diffusion 与 multi-stage topology；每个可恢复 case 后下一 case 从 awake + 开始；level-2 terminal case 之后不再复用 server；统计目标 lane 确实只初始化预期数量的 + engine，且 cleanup 后无 worker/device state 遗留。^[PR #5713] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5ade8ab3a..4762a4fc4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e788ef6e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0a7ac4c6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9ba838a32..107fcfda3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e788ef6e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 0a7ac4c6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2b826e5f1..f5d09180d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e788ef6e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0a7ac4c6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 62412689b..13d74f971 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e788ef6e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0a7ac4c6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 313fafaca..f33badcfe 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e788ef6e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0a7ac4c6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 040df59d9..750e340f9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e788ef6e` 验证存在 +- 源码校验：以上路径均已在 `main @ 0a7ac4c6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 7c2b5dcfb..9ee9a4e82 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5713", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -219,7 +219,10 @@ confidence: high - 禁止：错误 ACK 也清 tag；用 engine 全局禁令误伤已经支持 restore 的 diffusion worker。 - 验收：失败 ACK 保留 sleeping 状态；支持 level-2 的 diffusion 路径仍能 sleep → wake → generate，不支持的 stage 在调用 worker 前明确拒绝。 - ^[PR #4834] ^[PR #4905] ^[PR #4912] + assembled app 对不支持的 level-2 wake 保留 sleeping state，并把既有 + `NotImplementedError` 映射为 OpenAI-style structured HTTP 501；bare route mock 只证明 + exception propagation，不能代替这条 live contract。^[PR #4834] ^[PR #4905] + ^[PR #4912] ^[PR #5713] ### SERV-5c — 共享服务类重构必须闭合全部构造状态 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 64adef6bd..815984deb 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e788ef6e`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0a7ac4c6`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from f24a6165 \ - --to e788ef6e \ + --from e788ef6e \ + --to 0a7ac4c6 \ --repo \ --mode report-only ``` From d56816042f35c283af65305f522306edc673d7f2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 13:19:15 +0800 Subject: [PATCH 020/465] knowledge(vllm-omni): sync f201b717 (PR #5638) Upstream-Commit: f201b7171f7d3ae76ee9c4929dbd8718e284962d Upstream-PR: #5638 Knowledge-Delta: baseline-pins-and-minicpmo-code2wav-trt-contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minicpm-o-4-5/_index.md | 11 +++++--- .../vllm-omni/models/minicpm-o-4-5/rules.md | 26 +++++++++++++++++-- .../vllm-omni/models/step-audio2/_index.md | 6 ++--- .../models/step-audio2/architecture.md | 9 ++++--- 13 files changed, 53 insertions(+), 23 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 66163797e..423fc1eb4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e788ef6e76a93e8fe73c3072064f985ea0b50889 + previous_audited_sha: 0a7ac4c67e3061ef0a9b4a0283e90dd2981ea88c audited_ref: main - audited_sha: 0a7ac4c67e3061ef0a9b4a0283e90dd2981ea88c + audited_sha: f201b7171f7d3ae76ee9c4929dbd8718e284962d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2c973bf83..8126790c3 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 0a7ac4c67e3061ef0a9b4a0283e90dd2981ea88c`. The canonical machine baseline + f201b7171f7d3ae76ee9c4929dbd8718e284962d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4762a4fc4..3d3ce49bd 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0a7ac4c6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f201b717` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 107fcfda3..206d2ad39 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0a7ac4c6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f201b717` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f5d09180d..ae315fbd6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0a7ac4c6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f201b717` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 13d74f971..7a5deb9f9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0a7ac4c6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f201b717` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f33badcfe..ca1d87f1c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0a7ac4c6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f201b717` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 750e340f9..f1e996987 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0a7ac4c6` 验证存在 +- 源码校验：以上路径均已在 `main @ f201b717` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 815984deb..3aef5b44f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0a7ac4c6`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f201b717`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from e788ef6e \ - --to 0a7ac4c6 \ + --from 0a7ac4c6 \ + --to f201b717 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 79a66b786..6ae44a275 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -1,10 +1,10 @@ --- title: "MiniCPM-o 4.5" created: 2026-07-20 -updated: 2026-08-23 +updated: 2026-09-02 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/] +sources: ["PR #3642", "PR #5638", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/] confidence: high --- @@ -27,11 +27,16 @@ MiniCPM-o 多个版本共享通用 `MiniCPMO` architecture 名称，4.5 不能 runtime bridge 交给 TTS stage，再包装为 `OmniOutput.multimodal_outputs`；deploy 变体改变 资源拓扑，不改变数据合同。 +Code2Wav 在所有平台使用树内 `MiniCPMO45Token2wav`。CUDA 可显式开启 DiT estimator + +Campplus TensorRT；这是可关闭的局部加速，不替换 encoder/HiFT，也不会自动启用 +Step-Audio2 的 token2wav。engine cache/profile 与 fallback 门禁见 MCPMO-1c。 + 描述直达源码与模型专有门禁见 [rules](rules.md#direct-代码快速入口)；新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 ## 什么时候查这里 -- 审查 MiniCPM-o 4.5 registry、remote-code gate、TTS dependency、batch/stage handoff 或 native duplex session。 +- 审查 MiniCPM-o 4.5 registry、remote-code gate、TTS dependency、Code2Wav TensorRT、 + batch/stage handoff 或 native duplex session。 - 问题位于共享 bridge/batching 时转到 [Model Executor rules](../../components/model-executor/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 4a15f7325..47ef2ee14 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -1,10 +1,10 @@ --- title: "MiniCPM-o 4.5 规则" created: 2026-07-20 -updated: 2026-08-23 +updated: 2026-09-02 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #6154", "PR #6170", "PR #6318"] +sources: ["PR #3642", "PR #5638", "PR #6154", "PR #6170", "PR #6318"] confidence: high --- @@ -19,6 +19,7 @@ confidence: high | MiniCPM-o 4.5、`minicpmo_4_5`、版本识别 | MCPMO-2a | `config/pipeline_registry.py::OMNI_PIPELINES["minicpmo_4_5"]` → `model_executor/models/minicpmo_4_5/pipeline.py`；`model_executor/models/registry.py::_OMNI_MODELS` | | tokenizer/processor、`trust_remote_code` | MCPMO-1a | pipeline/config factory → `model_executor/models/minicpmo_4_5/` loader | | TTS extra、backend 初始化、空音频 | MCPMO-1b | `minicpmo_4_5_omni_tts.py::MiniCPMO45OmniTTSForConditionalGeneration`、`minicpmo_4_5_token2wav.py::MiniCPMO45Token2wav` | +| Code2Wav TensorRT、DiT/Campplus、engine cache/profile | MCPMO-1c | `minicpmo_4_5_code2wav.py::MiniCPMO45Code2Wav` → `batched_token2wav.py::BatchedToken2Wav._estimator_step`；共享实现 `step_audio2/step_audio2_dit_trt.py` | | batch、`runtime_info`、stage handoff | MCPMO-3a/3b | `stage_input_processors/minicpmo_4_5_omni.py` → `minicpmo_4_5_omni.py::MiniCPMO45OmniForConditionalGeneration` → TTS/code2wav | | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | | instructions/persona/voice/mode update、prefill slots、context lock | MCPMO-4b | `experimental/fullduplex/minicpmo45/session.py` → runtime adapter/session runner | @@ -39,6 +40,27 @@ confidence: high - 验收：缺 extra、坏配置和 backend 初始化失败分别得到可诊断错误；正常路径返回非空 且格式正确的音频。 ^[PR #3642] +## MCPMO-1c — Code2Wav TensorRT 是 CUDA-only 的显式可逆替换 + +- 触发：`token2wav_trt` / `MINICPMO_TOKEN2WAV_TRT=1`，或修改 Code2Wav 的 DiT、 + Campplus、ONNX/TRT engine cache 与 optimization profile。 +- 强制：所有平台都先构造树内 `MiniCPMO45Token2wav`；开关只在 CUDA 同时启用 DiT + estimator 与 Campplus 的 TensorRT 路径，非 CUDA 忽略开关，关闭开关保留完整 torch + 路径。DiT 只替换 `BatchedToken2Wav._estimator_step` 的逐步 estimator 调用，encoder 与 + HiFT 仍走 torch；engine 在调用者当前 CUDA stream 上执行。 +- 强制：强类型精度来自所选 dtype 的 ONNX graph，export 深拷贝已加载 DiT 后再 cast， + 不改变 live torch estimator 的 dtype。ONNX cache 是动态、可移植 graph；plan cache key + 必须精确包含 depth、dtype、`token2wav_trt_max_batch`、GPU SM 与 TensorRT version。 +- 强制：ONNX/plan 首次生成使用 PID 唯一临时文件并原子替换；step 在执行前检查 profile： + batch 上限为配置值且包含 classifier-free guidance 展开的 `2B`，chunk 固定上限 3000， + attention cache 固定上限 1000，越界给出可操作错误。Campplus 合同保持 `[T,80]` → + `[1,192]` fp32，并从 `_resolve_model_dir()` 已解析的本地模型目录取资产。 +- 禁止：把共享 `step_audio2_dit_trt.py` 的存在解释成 Step-Audio2 用户自动获得 TRT； + 当前开关只由 MiniCPM-o wiring 调用。也禁止把讨论中的通用 vocoder backend 当成已实现。 +- 验收：分别覆盖关闭/开启、非 CUDA 忽略、torch/TRT 数值与音频 parity、首次构建和 cache + 复用、不同 SM 不共享 plan、CFG 后 batch 越界以及 chunk/cache 越界。PR 未新增自动化测试， + 因而这些仍是后续改动必须补证的验证缺口。 ^[PR #5638] + ## MCPMO-2a — registry 使用 4.5 config/version predicate - 触发：pipeline auto-detection 看到通用 `MiniCPMO` architecture。 diff --git a/knowledge/repos/vllm-omni/models/step-audio2/_index.md b/knowledge/repos/vllm-omni/models/step-audio2/_index.md index fa45b5e90..54201d97c 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/_index.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/_index.md @@ -1,15 +1,15 @@ --- title: "Step-Audio2（音频 token 内嵌 LM 词表的语音对话）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py] +sources: ["PR #5638", vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py] --- # Step-Audio2 -以下事实在 `main @ 5d44868e` 复核。 +以下事实在 `main @ f201b717` 复核。 ## 名称与范围 diff --git a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md index 39d296340..04f6bf5d6 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md @@ -1,15 +1,15 @@ --- title: "Step-Audio2 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: architecture tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py] +sources: ["PR #5638", vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py] --- # Step-Audio2 架构 -事实在 `main @ 5d44868e` 复核;变体/入口速览见 [index](_index.md)。 +事实在 `main @ f201b717` 复核;变体/入口速览见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -25,6 +25,9 @@ sources: [vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vl s3tokenizer（prompt wav 语音 token）+ ONNX 说话人嵌入 + hyperpyyaml 加载的 flow-matching（10 步 ODE）+ flashcosyvoice HiFT 声码器 → 24 kHz;树内带说话人 wav——`assets/default_female.wav` 是默认,`default_male.wav` 是备选。 +- 共享加速实现：`step_audio2_dit_trt.py` 提供流式 DiT ONNX export/TRT stepper， + `step_audio2_token2wav.py` 提供 Campplus TRT helper；在此 pin 上只有 MiniCPM-o 的 + Code2Wav wiring 显式启用它们，Step-Audio2 pipeline 不会因共享代码存在而自动切换。 - 常量单一来源 `step_audio2_constants.py`：文本 ≤151688;音频 token **151696–158257**（`audio_vocab_size` 6562,相对 `audio_eos` 6561）;流式 `chunk_size 25` / `pre_lookahead_len 3` / mel cache 8 帧。 From d842fcb758e44831057d820c457fa19ee24eb689 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 13:25:00 +0800 Subject: [PATCH 021/465] knowledge(vllm-omni): sync 6a604be6 (PR #5608) Upstream-Commit: 6a604be6103423ae35fb55d9ead26c0dcccc225e Upstream-PR: #5608 Knowledge-Delta: baseline-pins-and-qwen3-tts-npu-rope-fallback --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/qwen3-tts/_index.md | 10 +++++--- .../repos/vllm-omni/models/qwen3-tts/rules.md | 24 +++++++++++++++++-- 11 files changed, 41 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 423fc1eb4..2637ab460 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0a7ac4c67e3061ef0a9b4a0283e90dd2981ea88c + previous_audited_sha: f201b7171f7d3ae76ee9c4929dbd8718e284962d audited_ref: main - audited_sha: f201b7171f7d3ae76ee9c4929dbd8718e284962d + audited_sha: 6a604be6103423ae35fb55d9ead26c0dcccc225e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8126790c3..d1b8627af 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - f201b7171f7d3ae76ee9c4929dbd8718e284962d`. The canonical machine baseline + 6a604be6103423ae35fb55d9ead26c0dcccc225e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3d3ce49bd..e9e09081c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f201b717` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6a604be6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 206d2ad39..978e0a248 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f201b717` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6a604be6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ae315fbd6..71d313056 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f201b717` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6a604be6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7a5deb9f9..b77f18038 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f201b717` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6a604be6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ca1d87f1c..6918aafe7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f201b717` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6a604be6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f1e996987..bf48d606b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f201b717` 验证存在 +- 源码校验：以上路径均已在 `main @ 6a604be6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3aef5b44f..2d96f7936 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f201b717`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6a604be6`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 0a7ac4c6 \ - --to f201b717 \ + --from f201b717 \ + --to 6a604be6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md index 06c704cf0..a598632f1 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", vllm_omni/entrypoints/openai/serving_speech.py] +sources: ["PR #5157", "PR #5608", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py] confidence: high --- @@ -12,12 +12,16 @@ confidence: high ## 名称与范围 -- 正式 owner：Qwen3-TTS serving、ref audio 与 artifact cache 请求语义。 +- 正式 owner：Qwen3-TTS serving、ref audio、artifact cache 请求语义与模型专有 NPU patch。 - serving 入口：`vllm_omni/entrypoints/openai/serving_speech.py`。 +- NPU tokenizer patch：`vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py`；共享 + RoPE layout helper 在 `vllm_omni/platforms/npu/layers/rotary_embedding.py`，此 pin 上只 + 证明 Qwen3-TTS consumer。 - 共享请求/cache 合同见 [Serving rules](../../components/serving/rules.md)。 ## 什么时候查这里 -- 审查 `x_vector_only_mode`、ICL、`ref_audio` artifact-only reuse 或 engine 存活性。 +- 审查 `x_vector_only_mode`、ICL、`ref_audio` artifact-only reuse、engine 存活性，或 NPU + tokenizer RoPE 的 BNSD/BSND shape fallback。 - 描述直达源码与具体不变量见 [Qwen3-TTS rules](rules.md#direct-代码快速入口)；模型家族结构见 [Qwen-Omni](../qwen-omni/_index.md)。 diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index e3d1bd0f2..6fce5e495 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -1,10 +1,10 @@ --- title: "Qwen3-TTS 规则" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-02 type: rule tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157"] +sources: ["PR #5157", "PR #5608"] confidence: high --- @@ -20,6 +20,7 @@ confidence: high | `ref_audio`、x-vector、ICL、artifact-only reuse | Q3TTS-1a/1b | `entrypoints/openai/serving_speech.py::_qwen3_tts_can_use_ref_audio_artifact_only`、`_track_ref_audio_artifact_warmup`、`_mark_ref_audio_artifact_ready_for_request` | | talker/code2wav、deferred audio codes | Model Executor / Scheduler 共享规则 | `model_executor/models/qwen3_tts/qwen3_tts_talker.py::Qwen3TTSTalkerForConditionalGeneration`、`qwen3_tts_code2wav.py::Qwen3TTSCode2Wav` | | OpenAI speech adapter | Q3TTS-1a/1b + Serving | `entrypoints/openai/tts_adapters/qwen3_tts.py::Qwen3TTSAdapter` → `serving_speech.py` | +| NPU、RoPE、BNSD/BSND、`codec_chunk_ramp` | Q3TTS-2a | `platforms/npu/models/qwen3_tts_tokenizer_v2.py::_apply_rotary_pos_emb_npu` → `platforms/npu/layers/rotary_embedding.py::npu_rotary_mul_with_bsnd_fallback` | ## Q3TTS-1a — ref-audio readiness 按 mode/capability 隔离 @@ -41,5 +42,24 @@ confidence: high - 验收：顺序测试证明 ICL → x-vector 只多一次重算、输出正确且后续同模式复用恢复； 若优化为 capability predicate，保留 x-vector artifact 不能服务 ICL 的单向边界。 ^[PR #5157] +## Q3TTS-2a — NPU RoPE 按真实 shape 在 BNSD 与 BSND fused path 间选择 + +- 触发：Qwen3-TTS 12 Hz tokenizer decoder 的 NPU RoPE，尤其启用短 + `codec_chunk_ramp` 且提高并发时的短序列 decode。 +- 强制：`unsqueeze_dim=1` 的输入合同是 rank-4 BNSD `[B,N,S,D]`。只有 + `D` 为偶数、element size 有效且整除 32、`(D/2) % (32/element_size) == 0`，并且 + `B*N <= S*8` 时，才保持 BNSD fused fast path，cos/sin 在 axis 1 扩维。 +- 强制：合法 rank-4 输入不满足上述 BNSD predicate 时，把 hidden states 以 + `transpose(1,2).contiguous()` 转成 BSND `[B,S,N,D]`，cos/sin 在 axis 2 扩维后调用同一 fused + `torch_npu.npu_rotary_mul`，再 transpose 回 `[B,N,S,D]`。q 和 k 必须使用同一 helper； + `unsqueeze_dim != 1` 的既有调用保持直接 fused 路径。 +- 禁止：让短序列/高并发 shape 继续进入会拒绝它的 BNSD tiler；也不能为规避崩溃而让 + 所有 shape 都承担 BSND transpose，丢掉已支持 shape 的 BNSD fast path。共享 helper 的 + 路径是模型无关的，但此 pin 只证明 Qwen3-TTS tokenizer consumer，不能外推到其他模型。 +- 验收：覆盖 `B*N == S*8` 不等式边界两侧、不同 dtype/alignment、rank-4 输入合同、非默认 + `unsqueeze_dim`，并把 BNSD fast path 与 BSND fallback 的 q/k 结果同 eager RoPE 做数值 + parity。当前测试只把一个 unsupported shape 的 cos/sin dispatch 期望改为 BSND axis， + 尚无专门的 supported/unsupported 边界矩阵或 eager 数值 parity。 ^[PR #5608] + 共享 readiness/错误隔离规则见 [Serving rules](../../components/serving/rules.md)； Qwen 家族入口见 [Qwen-Omni](../qwen-omni/_index.md)。 From b58c266d14d22e26ca979b000ff63bfd373fb423 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 13:35:03 +0800 Subject: [PATCH 022/465] knowledge(vllm-omni): sync ae7bee0c (PR #5752) Upstream-Commit: ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783 Upstream-PR: #5752 Knowledge-Delta: MiniMax H3 official input matrix and serving media contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules.md | 19 +++++-- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../vllm-omni/models/minimax-h3/_index.md | 13 ++--- .../vllm-omni/models/minimax-h3/rules.md | 53 ++++++++++++++++++- 12 files changed, 87 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2637ab460..0f5d4ed58 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f201b7171f7d3ae76ee9c4929dbd8718e284962d + previous_audited_sha: 6a604be6103423ae35fb55d9ead26c0dcccc225e audited_ref: main - audited_sha: 6a604be6103423ae35fb55d9ead26c0dcccc225e + audited_sha: ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d1b8627af..78844756e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 6a604be6103423ae35fb55d9ead26c0dcccc225e`. The canonical machine baseline + ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e9e09081c..c1ec91784 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6a604be6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ae7bee0c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 978e0a248..119dd960a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6a604be6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ae7bee0c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 71d313056..6321e39c0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6a604be6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ae7bee0c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b77f18038..1cf56c054 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6a604be6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ae7bee0c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6918aafe7..3f0736c3d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6a604be6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ae7bee0c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index bf48d606b..06439e7b1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6a604be6` 验证存在 +- 源码校验：以上路径均已在 `main @ ae7bee0c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 9ee9a4e82..71b8ae69f 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5713", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5713", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -25,7 +25,7 @@ confidence: high | sleep/wake、partial stage/tag、idempotency、ACK、generation admission | `engine-lifecycle`：`SERV-5a`, `SERV-5b` | `entrypoints/async_omni.py::{AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni.generate}` → `worker/base.py::{handle_sleep_task,handle_wake_task}` / `diffusion/worker/diffusion_worker.py` | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | -| video reference 解码、frame selection、decoder backend、空帧 | `media-ingress`：`SERV-1c` | `entrypoints/openai/video_api_utils.py::{OmniVideoBackend,_decode_video_bytes}` → video server callers | +| video reference 解码、mixed media、capability、bounded upload、空帧 | `media-ingress`：`SERV-1c/1d` | `entrypoints/openai/video_api_utils.py::{OmniVideoBackend,_decode_video_bytes}`、`serving_video.py::supports_mixed_reference_inputs` → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | | Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle | `metrics-lifecycle`：`SERV-2a`, `SERV-2b` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | @@ -33,7 +33,7 @@ confidence: high |---|---|---| | `core` | 每次 serving 审查 | `SERV-4c` | | `streaming-format` | SSE、audio streaming、format/default/capability | `SERV-1a`, `SERV-1b` | -| `media-ingress` | video reference、decoder registry/backend、first/last frames | `SERV-1c` | +| `media-ingress` | video reference、decoder registry/backend、mixed capability、bounded upload | `SERV-1c`, `SERV-1d` | | `metrics-lifecycle` | metrics、gauge、replica、collector | `SERV-2a`, `SERV-2b` | | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c` + 命中模型规则 | @@ -76,6 +76,19 @@ confidence: high `keep`、非正 `max_frames`、无效字节与 loader 零帧，且真实 Qwen multimodal/video 请求仍通过公开入口。 ^[PR #5085] +## SERV-1d — mixed media 必须 capability-gated，request 错误在解码前限界 + +- 触发：共享 video serving 接受 mixed image/video/audio、typed URL/data URL 或 multipart upload。 +- 强制：unknown model 默认无 mixed capability；只有 model metadata 明示支持才允许 image+video。 + multipart 大小/格式等可在字节边界判定的约束必须 bounded-read 并在解码前拒绝；pipeline 的 + request-facing validation 使用可跨 worker 保留 client metadata 的错误类型，统一映射为 400。 +- 禁止：删除旧互斥 guard 后让所有 diffusion 模型继承 mixed input；用 unbounded `read()` 后 + 再检查；让 plain `ValueError` 穿过 stage worker 变成 500。 +- 验收：支持模型的 mixed request 成功，unknown/不支持模型在 engine 前返回 400；multipart + oversize/bad format 与 pipeline count/shape 错误保持同一 client-error 合同。typed URL/data URL + 是否同样 bounded 必须单独证明；本 pin 的 helper 仍完整读取。模型专有 allowlist、temp-file + 与 input matrix 见 [MiniMax H3 rules](../../models/minimax-h3/rules.md)。 ^[PR #5752] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2d96f7936..af6f9186c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6a604be6`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ae7bee0c`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from f201b717 \ - --to 6a604be6 \ + --from 6a604be6 \ + --to ae7bee0c \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index b3aca020e..9b844e590 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5709", "PR #5737", "PR #5740", "PR #5756", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -16,10 +16,11 @@ confidence: high `MiniMaxH3Pipeline`，实现位于 `diffusion/models/minimax_h3/`。 - 支持 `t2va`、`fl2va`、`ref2va` 三种 joint video+audio 条件模式；一个 server 进程 一次只加载 `FL2VA` 或 `Ref2VA` 分区，不能把两个分区当成同一份权重同时服务。 -- 输出是带同步音频的 MP4；参考视频、图像和音频的预处理合同由 pipeline 与 video API - 共同决定，不能仅凭 endpoint 名称推断输入组合。 -- 生成合同固定为 24 FPS 视频与 32 kHz 音频；空间尺寸必须按 32 对齐，宽高比限制在 - `1:4` 到 `4:1`，这些约束在 request validation 阶段执行而不是由 VAE 静默修正。 +- 输出是带同步音频的 MP4；FL2VA 接受首帧、尾帧或有序首尾帧，Ref2VA 支持 image-only + 及有 visual reference 的 image/video/audio mixed matrix。完整计数、媒体和 API 合同见 + MMH3-2a/2b。 +- 生成合同固定为 24 FPS 视频与 32 kHz 音频；官方输出 duration、named ratio、768 + short-edge 与 32-pixel canvas policy 在 request validation 阶段执行，而不是由 VAE 静默修正。 ## 并行与加载约束 @@ -62,5 +63,5 @@ PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、t ## 审查入口 -H3 online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 +H3 input matrix/media ingress 以及 online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 [MiniMax H3 rules](rules.md#direct-代码快速入口)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index d90b79b32..90151088e 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5737", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, recipes/MiniMaxAI/MiniMax-H3.md] +sources: ["PR #5737", "PR #5752", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md] confidence: high --- @@ -19,6 +19,8 @@ confidence: high | online FP8、`ignored_layers`、component prefix | `MMH3-1a` | `pipeline_minimax_h3.py::_resolve_component_quant_config` → `MiniMaxH3DiTModel` linear prefix | | grouped QKV、fused MLP、weight loader、TP | `MMH3-1a` | `minimax_h3_transformer.py::MiniMaxH3DiTModel.load_weights` → active vLLM loader | | FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | +| FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | +| media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | ## MMH3-1a — component namespace 与 checkpoint transform 必须在 active loader 前闭合 @@ -50,5 +52,54 @@ confidence: high RMS ratio 在 0.50–2.00，且 sample rate 为 32000；单测还要证明 phase-tolerant metric 接受相移但拒绝 spectral drift。 ^[PR #5737] +## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 + +- 触发：H3 `t2va`/`fl2va`/`ref2va`、reference 计数、frame index、canvas、duration 或 + `num_outputs_per_prompt`。 +- 强制：FL2VA 接受一张首/尾帧或两张有序首尾帧，signature 仅为 `[0]`、`[-1]`、 + `[0,-1]`，未显式给出时一张默认 `[0]`、两张默认 `[0,-1]`。Ref2VA 必须至少有一项 + image/video visual reference；image≤9、video≤3、standalone audio≤3、总数≤12，允许 + image-only 及 image/video/audio mixed matrix，但拒绝 audio-only。 +- 强制：输出固定 24 FPS、32 kHz audio、duration 4–15 秒。省略 width/height 时 + `short_edge` 只能为 768；T2VA 必须显式选择 `21:9`、`16:9`、`4:3`、`1:1`、`3:4`、 + `9:16`，FL2VA 跟随首图 geometry 并忽略通用 ratio override，Ref2VA 默认 `16:9`， + `adaptive`/`auto` 是该默认的 alias。 +- 强制：`num_outputs_per_prompt` 仅为 1–10，各输出 seed 是 `seed + output_index`；pipeline + tensor output 与未被公开 caller 使用的 `generate_videos()` 保留 fan-out。此 pin 上 public + async job 与 `/v1/videos/sync` 都调用 `generate_video_bytes()`，多输出时告警并只持久化/返回 + `artifacts.videos[0]`；不得把 recipe 的“async 返回全部”描述当成实现证据。 +- 禁止：把 partition 名当输入能力；FL2VA/Ref2VA 权重分区仍需与 task 匹配。不得把长 + reference clip 截成输出长度；reference 自己的 segment/duration 合同由 MMH3-2b 约束。 +- 验收：以 F1/F2、R1–R10、G1–G4 matrix 覆盖正反例，包含 mixed reference、audio-only、 + ratio/default、duration 边界、pipeline fan-out、seed 顺序与两个 public endpoint 的 first-only + 行为；文档与实现冲突必须显式暴露。 ^[PR #5752] + +## MMH3-2b — media ingress 在解码前限界，request 错误保持 HTTP 400 + +- 触发：typed `image_reference`/`video_reference`/`audio_reference`、multipart + `input_reference(s)`、URL/data URL 下载、ffprobe/transcode 或 `start_time_seconds`。 +- 强制：typed 字段接受单对象或有序 list；multipart 按 MIME/suffix 分类并保持每种媒体的 + 顺序。仅 H3 multipart `input_reference(s)` 在读取/解码前执行 bounded read 与 allowlist： + image≤30 MiB（JPEG/PNG/WEBP/HEIC/HEIF）、 + video≤50 MiB（MP4/MOV，H.264/H.265，若有 audio 则 AAC/MP3）、audio≤15 MiB（WAV/MP3， + PCM/MP3）。image/video dimension 为 256–5760、ratio 为 0.4–2.5；video FPS 23.976–60。 +- 强制：每个 video/audio reference 时长 2–15 秒；video 总时长≤15 秒，embedded video + audio 与 standalone audio 合并后的 conditioning 总时长也≤15 秒。每个 video 的 + `start_time_seconds` 必须落在 source duration 内并剩余至少 2 秒。只容忍官方双视频 + container probe 的微小 rounding overflow，并裁到剩余总时长；真实 overflow 仍拒绝。 +- 强制：typed URL/data-URL video 必须 materialize 为 request-scoped 临时 MP4 source，再交给 + 只收 path 的 preparation，成功/异常都清理；path-backed typed video/audio 在下载后由模型 + preparation 检查 file metadata。mixed image+video 只有 model metadata 明示 capability 才放行； + 分布式非零 DiT rank 用已广播的全局 `video_count` 判断 visual 是否存在。 +- 禁止：PIL/ffmpeg 转换后才检查原始格式/大小；把 request-facing `ValueError` 变成 HTTP 500； + 用 fake engine 的成功代替 source-path、cleanup 或 cross-model capability 证明。 +- 验收：非法计数、multipart 格式/大小及 path-backed media metadata/segment 均在生成前返回 + `OmniClientError`/HTTP 400；覆盖 typed URL/data URL、multipart mixed、unsupported model、 + temp cleanup 和多 rank video+audio。此 pin 的 typed URL/data URL helper 仍先完整 + download/base64 decode；typed image + 还会丢失原始 size/format，因此 multipart 的 bounded/predecode 证明不可外推，属于验证与 + hardening 缺口。API server 中的 H3 专用 helper 也仍有 reviewer 指出的后续泛化重构债务。 + ^[PR #5752] + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 From 943758f5774b3dd2c2bc927fc1450d7da7bb1818 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 13:41:22 +0800 Subject: [PATCH 023/465] knowledge(vllm-omni): sync 740cb35a (PR #5635) Upstream-Commit: 740cb35a595409fdd7c081ff9d4ebbaeb6520e39 Upstream-PR: #5635 Knowledge-Delta: MOSS-TTS codec generation and checkpoint topology contract --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../repos/vllm-omni/models/moss-tts/_index.md | 9 ++-- .../vllm-omni/models/moss-tts/architecture.md | 10 +++-- .../repos/vllm-omni/models/moss-tts/rules.md | 42 +++++++++++++++++++ 12 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/moss-tts/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0f5d4ed58..4e5d8cda2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6a604be6103423ae35fb55d9ead26c0dcccc225e + previous_audited_sha: ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783 audited_ref: main - audited_sha: ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783 + audited_sha: 740cb35a595409fdd7c081ff9d4ebbaeb6520e39 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 78844756e..736c4f91b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783`. The canonical machine baseline + 740cb35a595409fdd7c081ff9d4ebbaeb6520e39`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c1ec91784..fb16e4ab6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ae7bee0c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 740cb35a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 119dd960a..883ddbf57 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ae7bee0c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 740cb35a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6321e39c0..0345d880a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ae7bee0c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 740cb35a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1cf56c054..56ac2dfd7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ae7bee0c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 740cb35a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3f0736c3d..818011193 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ae7bee0c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 740cb35a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 06439e7b1..cd6a95d47 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ae7bee0c` 验证存在 +- 源码校验：以上路径均已在 `main @ 740cb35a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index af6f9186c..130395709 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ae7bee0c`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 740cb35a`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 6a604be6 \ - --to ae7bee0c \ + --from ae7bee0c \ + --to 740cb35a \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/moss-tts/_index.md b/knowledge/repos/vllm-omni/models/moss-tts/_index.md index 1693c3c3f..fe71f5d11 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/_index.md @@ -1,15 +1,15 @@ --- title: "MOSS-TTS 家族（Delay/Realtime/Local/Nano,一族八 deploy）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/model_executor/models/moss_tts/, vllm_omni/model_executor/models/moss_tts_nano/, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/] +sources: ["PR #5635", vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/model_executor/models/moss_tts/, vllm_omni/model_executor/models/moss_tts_nano/, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/] --- # MOSS-TTS 家族 -以下事实在 `main @ 5d44868e` 复核。 +以下事实在 `main @ 740cb35a` 复核。 ## 名称与范围 @@ -48,6 +48,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | delay 生命周期、伪文本 logits、双代 tokenizer | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | +| codec v1/v2 选择、projection/checkpoint topology | [rules](rules.md#direct-代码快速入口) | loader 门禁与测试缺口 | ## 配置与 checkpoint 差异（8 份 deploy 速查） @@ -68,6 +69,6 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline ## 什么时候查这里 -- 审查任一 MOSS 变体的码流、CUDA-graph 开关或 serving 适配;新增变体时先对 +- 审查任一 MOSS 变体的码流、codec checkpoint 代际、CUDA-graph 开关或 serving 适配;新增变体时先对 这 8 份 YAML 的差异矩阵。 - 语义验收见 [model-validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md index ff6194d5e..66b4125ed 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md @@ -4,12 +4,12 @@ created: 2026-07-21 updated: 2026-09-02 type: architecture tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, vllm_omni/model_executor/models/moss_tts/pipeline.py, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/moss_voice_generator.yaml] +sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, vllm_omni/model_executor/models/moss_tts/pipeline.py, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/moss_voice_generator.yaml] --- # MOSS-TTS 架构 -事实在 `main @ 78c144f3` 复核;变体/deploy 速查见 [index](_index.md)。 +事实在 `main @ 740cb35a` 复核;变体/deploy 速查见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -24,7 +24,9 @@ sources: [vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, `text_lm_head` 加载但不用（binary 停止头判停）。 - 共享 codec `MossTTSCodecDecoder`：**同一解码器里装两代 audio tokenizer** （v1 24 kHz;v2 48 kHz 立体声,带 `RingKVCache` 流式基建）+ 两个 CUDA-graph - wrapper（批解码 vs 流式解码）;按 codec checkpoint config 运行期选代。 + wrapper（批解码 vs 流式解码）;按 raw codec config 的 `number_channels` 运行期选代， + 字段缺失/单通道走 v1，≥2 走 v2。projection 的 Linear/Identity topology 同样属于 + checkpoint ABI，具体门禁见 [rules](rules.md)。 - 共享框架面：[Config 组件](../../components/configuration/architecture.md)、 SharedMemoryConnector、speaker cache（`reference_encoder.py`,树内注明援引 Fish/CosyVoice3/Qwen3-TTS 先例）。 @@ -67,4 +69,4 @@ sources: [vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, 权威映射）;recipe `recipes/OpenMOSS/MOSS-TTS.md`。 - 已知未决：delay 路下 connector 级 `codec_chunk_frames` 是否还有残余作用 未追;`moss_tts_local_codec` 与 `moss_tts_codec` 在 serving 侧的分流细节 - 未逐行读;v1/v2 tokenizer 运行期选代逻辑未细读。 + 未逐行读。v1/v2 选择与 projection topology 已复核，但 pin 上没有对应自动化 regression。 diff --git a/knowledge/repos/vllm-omni/models/moss-tts/rules.md b/knowledge/repos/vllm-omni/models/moss-tts/rules.md new file mode 100644 index 000000000..42587248b --- /dev/null +++ b/knowledge/repos/vllm-omni/models/moss-tts/rules.md @@ -0,0 +1,42 @@ +--- +title: "MOSS-TTS 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, model-executor] +sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py] +confidence: high +--- + +# MOSS-TTS 规则 + +只有 `MOSSTTS-数字字母` 是可审计规则 ID。 + +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | 第一批源码 | +|---|---|---| +| codec v1/v2、`number_channels`、missing weights | `MOSSTTS-1a` | `modeling_moss_tts_codec.py::MossTTSCodecDecoder._build_codec` → Stage-1 weight loader | +| `_ProjectedTransformer`、`in_proj`/`out_proj`、Identity | `MOSSTTS-1a` | `audio_tokenizer.py::_ProjectedTransformer` → codec checkpoint parameter names | + +## MOSSTTS-1a — codec 代际与 module topology 必须由 checkpoint 原始结构决定 + +- 触发：MOSS Stage-1 codec config 解析、v1/v2 类选择、vendored tokenizer module 或 + checkpoint missing/shape-mismatch validation。 +- 强制：先用 `MossAudioTokenizerV2Config.get_config_dict(codec_path)` 读取 raw config；只有 + raw `number_channels >= 2` 才尝试 v2 config/model。字段缺失按 `1` 判为 v1，直接使用 + `MossAudioTokenizerConfig`/`MossAudioTokenizerModel`，不能让两代共享的 `model_type` 或 v2 + config 的默认双通道值替 checkpoint 决定代际。 +- 强制：v2 config/model 构造失败时记录异常并显式回退 v1；raw config 读取本身在选择前完成， + 其失败不得伪装成某一代成功。`_ProjectedTransformer` 的 `in_proj` 仅在 + `input_dimension != d_model` 时为无 bias `Linear`，否则为 `Identity`；`out_proj` 对 + `output_dimension != d_model` 使用同一规则。module topology 与 checkpoint parameter 集必须一致。 +- 禁止：总是先实例化 v2 来“探测”版本；用偶然通过的 frame-rate validation 证明类正确； + dimensions 相等时额外创建 checkpoint 不含的 learned projection，或无条件用 Identity + 丢掉 dimensions 不等时的权重。 +- 验收：覆盖 raw config 字段缺失/单通道/v2 双通道、v2 构造失败回退、equal/unequal input + 与 output dimensions，并断言 model class、projection 类型、参数名、weight completeness 和 + forward shape。PR 只提供 Ascend NPU 手工 load/audio 验证，未新增这些自动化回归。 ^[PR #5635] + +共享 loader 的 dtype/config 最小读取合同见 +[Model Executor rules](../../components/model-executor/rules.md)。 From 3799993949168d874161185ed6b696120c2c8157 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 13:48:24 +0800 Subject: [PATCH 024/465] knowledge(vllm-omni): sync 6510de34 (PR #5802) Upstream-Commit: 6510de344ebff729e9e066bef1a23ff120ce422c Upstream-PR: #5802 Knowledge-Delta: heterogeneous-block DLO AllGather buffer contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 20 +++++++++++++------ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 26 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4e5d8cda2..5c17fbf39 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783 + previous_audited_sha: 740cb35a595409fdd7c081ff9d4ebbaeb6520e39 audited_ref: main - audited_sha: 740cb35a595409fdd7c081ff9d4ebbaeb6520e39 + audited_sha: 6510de344ebff729e9e066bef1a23ff120ce422c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 736c4f91b..b70fab687 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 740cb35a595409fdd7c081ff9d4ebbaeb6520e39`. The canonical machine baseline + 6510de344ebff729e9e066bef1a23ff120ce422c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fb16e4ab6..c2d05c108 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 740cb35a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6510de34` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 883ddbf57..aca52d638 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 740cb35a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6510de34` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index bc45488ae..6cc162b7a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -1,10 +1,10 @@ --- title: "Diffusion 共享规则" created: 2026-07-20 -updated: 2026-08-23 +updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5737", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5737", "PR #5802", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/, tests/diffusion/offloader/test_distributed_layerwise_backend.py] confidence: high --- @@ -25,6 +25,7 @@ confidence: high | ModelOpt/checkpoint adapter、weight/scale remap、unknown tensor、resolution path | `checkpoint-distributed`：`DIFF-2a` | `diffusers_loader.py::{_get_checkpoint_adapter,load_weights}` → `modelopt.py::{_resolve_target_and_output_names,adapt}` | | host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | | HSDP/FSDP、`fully_shard`、DeviceMesh、packed/scalar parameter、FP8 | `checkpoint-distributed`：`DIFF-2b` | `distributed/hsdp.py::{apply_hsdp_to_model,shard_model}` → loader `_load_model_with_hsdp` → `hsdp_fp8.py` | +| distributed layerwise offload、AllGather、异构 block、shared buffer | `checkpoint-distributed`：`DIFF-2e` | `offloader/distributed_layerwise_backend.py::{DistributedLayerwiseOffloadHook.initialize_hook,prefetch_layer,DistributedLayerwiseOffloadBackend._allocate_shared_buffers}` | | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `data.py::_propagate_quantization_from_tf_config` → loader weight sources/post-load → component linear consumer | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | paged KV/cache、backend/platform、GQA/layout、预算与 admission | `system-runtime`：`DIFF-4a`–`4f` | engine init → `diffusion_kv/{initialization,paged_attention_adapter}.py` → platform hook → scheduler | @@ -34,7 +35,7 @@ confidence: high |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d` | | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、async output readiness | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity | `DIFF-2a`, `DIFF-2b`, `DIFF-2c`, `DIFF-2d` | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload | `DIFF-2a`, `DIFF-2b`, `DIFF-2c`, `DIFF-2d`, `DIFF-2e` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4f` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | @@ -141,11 +142,18 @@ confidence: high `OffloadPlan` 或 block discovery。 - 强制：明确 rank 本地 CPU shard、固定双 GPU buffer、H2D/AllGather stream 和模型 block list 的 owner；`dlo_use_allgather=false` 走每 rank full-weight 路径，开启 - AllGather 时不得再叠加已 DTensor-sharded 的 HSDP 参数。 + AllGather 时不得再叠加已 DTensor-sharded 的 HSDP 参数。共享 output buffer 可按所有 hook + 的最大 block 分配，但每个 hook 调 collective 前必须按 dtype 截成自己的 + `_ag_output_sizes[dtype] == dp_size * local_shard.numel()` 前缀；block-flat-relative repoint + offset 只能在这段实际 block buffer 内解释。 - 禁止：用 heuristic 找不到 block 时静默假装 offload 已启用；对 HSDP 参数二次分片； - 把 CPU 内存、AllGather 同步和设备显存开销隐藏在一个泛化的 `enable_cpu_offload` 开关。 + 把 CPU 内存、AllGather 同步和设备显存开销隐藏在一个泛化的 `enable_cpu_offload` 开关； + 把 max-sized shared buffer 原样交给较小 block 的 `all_gather_into_tensor`。 - 验收：CPU/Gloo 单 rank 覆盖 DTensor wrapper、shard padding、双 buffer 和 disable - cleanup；配置测试覆盖 AllGather+HSDP 的明确失败及 no-AllGather 的允许路径。 + cleanup；配置测试覆盖 AllGather+HSDP 的明确失败及 no-AllGather 的允许路径。另以至少两个 + 不同 block size 验证 small/large hook 都满足 collective size equality、重建权重和 repoint。 + 当前回归用 mocked collective 检查 8/32-element、dp=2 合同；没有真实 multi-rank collective + 覆盖，PR 的 8×NPU H3 E2E 说明也缺少已填写的 commit 证据，不能升级为验证结论。 ^[PR #5802] ## 质量阈值与资源辅助 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0345d880a..13e6a3b5a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 740cb35a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6510de34` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 56ac2dfd7..f63e95f1f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 740cb35a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6510de34` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 818011193..1ec708829 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 740cb35a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6510de34` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cd6a95d47..6d26ed18f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 740cb35a` 验证存在 +- 源码校验：以上路径均已在 `main @ 6510de34` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 130395709..c01e315d7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 740cb35a`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6510de34`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from ae7bee0c \ - --to 740cb35a \ + --from 740cb35a \ + --to 6510de34 \ --repo \ --mode report-only ``` From 2d7ca91a4cd9f87f9644a50b028ec984cdf9f231 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 13:58:13 +0800 Subject: [PATCH 025/465] knowledge(vllm-omni): sync 7fb62db5 (PR #5402) Upstream-Commit: 7fb62db5c226558162dbee5b204d8d57479d28a0 Upstream-PR: #5402 Knowledge-Delta: hardware-nested DFX baseline artifact contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 19 ++++++++++++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 30 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5c17fbf39..133160c35 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 740cb35a595409fdd7c081ff9d4ebbaeb6520e39 + previous_audited_sha: 6510de344ebff729e9e066bef1a23ff120ce422c audited_ref: main - audited_sha: 6510de344ebff729e9e066bef1a23ff120ce422c + audited_sha: 7fb62db5c226558162dbee5b204d8d57479d28a0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b70fab687..999ddd3b4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 6510de344ebff729e9e066bef1a23ff120ce422c`. The canonical machine baseline + 7fb62db5c226558162dbee5b204d8d57479d28a0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 2c12423d3..84e20105d 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] +sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] confidence: high --- @@ -21,6 +21,7 @@ confidence: high | pre-commit、SPDX、shellcheck、stability marker | `OMNI-CI-2a` | `.pre-commit-config.yaml`、`.buildkite/**`、`tools/**` | | xdist、共享 worker、下载 fixture、进程池 | `OMNI-CI-2b` | `tests/conftest.py`、`tests/helpers/**`、`tests/model_tests/**` | | 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | +| perf baseline、hardware label、DFX result artifact、assert-baseline | `OMNI-CI-3a` | `tests/dfx/conftest.py`、`tests/dfx/perf/scripts/run_benchmark.py`、`run_diffusion_benchmark.py`、`tests/dfx/perf/tests/**` | ## OMNI-CI-1a — 硬件 lane 必须真实收集并执行目标路径 @@ -88,3 +89,19 @@ confidence: high - 验收：分别覆盖 LLM、diffusion 与 multi-stage topology；每个可恢复 case 后下一 case 从 awake 开始；level-2 terminal case 之后不再复用 server；统计目标 lane 确实只初始化预期数量的 engine，且 cleanup 后无 worker/device state 遗留。^[PR #5713] + +## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 + +- 触发：修改 perf JSON 的 `baseline`、硬件 label/marker、benchmark result schema，或恢复性能阈值断言。 +- 强制：当前 runner 用 `copy.deepcopy` 将输入中的原始 `baseline` 对象原样写入 result artifact； + 硬件 marker/resource label 只路由执行，runner 不按它选择 baseline，也不按 sweep/concurrency + 解析阈值。新配置应使用 `{hardware: {metric: threshold}}` 的硬件嵌套结构，并把缺失 bucket + 当作显式缺口；下游报告工具可以另行选择 bucket，但须声明选择合同。 +- 禁止：把 result 中存在 baseline 描述成 active regression gate；从 `npu: "A3"` marker 推断存在 + A3 阈值；声称当前 runner 实现了文档所说的“按 runtime hardware label 选择”或 flat baseline + 兼容。目标 pin 中这些文档措辞是滞后/前瞻描述：选择与断言 helper、`--assert-baseline` 和 + `skip-performance-assertion` 已删除，flat 输入至多会被不透明地复制，未被 runner 解释。 +- 验收：当前 CI 只用 `completed == num_prompt(s)` gate 请求完成数，baseline artifact round-trip + 保持输入结构和值；若重新引入性能 gate，必须有显式 consumer、硬件 bucket 选择、metric + directionality 与失败测试。目标 pin 的已迁移 baseline bucket 全是 H100；九个 A3-marked 配置 + 没有任何 A3 baseline bucket，这是待补 gap，不是 NPU 性能 gate。^[PR #5402] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c2d05c108..b1b1c1242 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6510de34` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7fb62db5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index aca52d638..519055362 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6510de34` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 7fb62db5` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 13e6a3b5a..15fb9e5ca 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6510de34` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7fb62db5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f63e95f1f..3502747ec 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6510de34` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7fb62db5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1ec708829..72dfb433a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6510de34` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 7fb62db5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6d26ed18f..3f9539172 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6510de34` 验证存在 +- 源码校验：以上路径均已在 `main @ 7fb62db5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c01e315d7..ebef80cd5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6510de34`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7fb62db5`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 740cb35a \ - --to 6510de34 \ + --from 6510de34 \ + --to 7fb62db5 \ --repo \ --mode report-only ``` From 4fbf8ab91a723f9443152c3274a4e7e24269cbca Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 14:07:53 +0800 Subject: [PATCH 026/465] knowledge(vllm-omni): sync 2eada79b (PR #5703) Upstream-Commit: 2eada79b99ae64acd835e5ca3794eb6ff60f4424 Upstream-PR: #5703 Knowledge-Delta: MiniMax H3 conditioned VAE device RNG contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minimax-h3/_index.md | 9 ++++++-- .../vllm-omni/models/minimax-h3/rules.md | 22 ++++++++++++++++++- 11 files changed, 40 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 133160c35..eb848d1f7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6510de344ebff729e9e066bef1a23ff120ce422c + previous_audited_sha: 7fb62db5c226558162dbee5b204d8d57479d28a0 audited_ref: main - audited_sha: 7fb62db5c226558162dbee5b204d8d57479d28a0 + audited_sha: 2eada79b99ae64acd835e5ca3794eb6ff60f4424 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 999ddd3b4..a1ecb2d21 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 7fb62db5c226558162dbee5b204d8d57479d28a0`. The canonical machine baseline + 2eada79b99ae64acd835e5ca3794eb6ff60f4424`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b1b1c1242..c7c0cdac6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7fb62db5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2eada79b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 519055362..45dfef0cd 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7fb62db5` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 2eada79b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 15fb9e5ca..f158d4ef8 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7fb62db5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2eada79b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3502747ec..3bcac34de 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7fb62db5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2eada79b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 72dfb433a..f7718013c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7fb62db5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2eada79b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3f9539172..aae20fbc7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7fb62db5` 验证存在 +- 源码校验：以上路径均已在 `main @ 2eada79b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ebef80cd5..70662ff3d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7fb62db5`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2eada79b`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 6510de34 \ - --to 7fb62db5 \ + --from 7fb62db5 \ + --to 2eada79b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 9b844e590..5edd3bfb9 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -41,6 +41,9 @@ confidence: high - 音频加载优先使用 torchaudio；当 TorchCodec/torchaudio 在 CPU-only aarch64 环境不可用 时，`reference_video.load_audio_file` 回退到 soundfile，再对 libsndfile 不支持的格式 通过 ffmpeg 转 WAV。该回退保持 `(channels, samples)` float32 与原始 sample rate 合同。 +- conditioned image/video VAE 用固定内部 seed，并在 `fork_rng` 中播种 CPU 与参数所在设备后 + 恢复 state；MUSA recipe 记录 MTT S5000 验证。目标实现实际接纳所有非 CPU device，而实机 + RNG 证据只覆盖 CUDA/MUSA；支持边界与并发缺口见 MMH3-2c。 ## ComfyUI 请求路由 @@ -58,7 +61,9 @@ T2VA full-model accuracy 入口在 `tests/e2e/accuracy/minimax_h3/`：模型 sna PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、text-encoder TP4 和 VAE patch parallel 4。该用例是精度/媒体合同，不是性能基线。 硬件 recipe 只记录已验证的 GPU/NPU 形状；性能数字不能从 recipe 的配置示例泛化为全硬件 -保证。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 +保证。目标 pin 的 MUSA recipe 把 launch 描述成 validated Ref2VA 配置，却在命令中导出 +`MODEL="${MODEL_ROOT}/FL2VA"`；按分区合同它不能服务 `task=ref2va`，执行示例前必须改为 +`Ref2VA` 分区，不能把原 snippet 当成可运行的 Ref2VA profile。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 [Configuration](../../components/configuration/_index.md) 和 [Serving](../../components/serving/_index.md)。 ## 审查入口 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 90151088e..02cf0e98c 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5737", "PR #5752", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md] +sources: ["PR #5703", "PR #5737", "PR #5752", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] confidence: high --- @@ -21,6 +21,7 @@ confidence: high | FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | | FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | +| conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | ## MMH3-1a — component namespace 与 checkpoint transform 必须在 active loader 前闭合 @@ -101,5 +102,24 @@ confidence: high hardening 缺口。API server 中的 H3 专用 helper 也仍有 reviewer 指出的后续泛化重构债务。 ^[PR #5752] +## MMH3-2c — conditioned VAE 的固定种子必须按实际设备隔离并恢复 + +- 触发：修改 H3 image/video reference 的 VAE encode、固定 keyframe seed、`fork_rng`、设备 + generator 或 accelerator backend 支持。 +- 强制：`encode_image()`/`encode_video()` 在采样前暂时把 VAE 转成 FP32，在 + `torch.random.fork_rng(devices=...)` 内同时播种 CPU default generator 与 active-device + generator，退出时恢复 RNG state，并在 `finally` 恢复原 dtype；image path 还必须恢复 + `parallel_tiling`。这个 seed 是 conditioned VAE 的固定内部 seed，不能描述成 request seed。 +- 禁止：把 PR 文本中的 CUDA+MUSA allowlist 当成目标实现。目标代码实际以 + `parameter.device.type != "cpu"` 为条件，把所有非 CPU device 交给 `torch.get_device_module()`； + 现有实机证据只覆盖 CUDA/MUSA，不能外推 XPU、NPU 或 ROCm RNG 语义。也不能因 + `fork_rng` 最终恢复 state 就宣称并发安全：context 内仍会暂时改写 process-global CPU/device + generator，重叠调用需要序列化或独立并发证明。 +- 验收：同 seed 的 image/video condition latent 可重复，正常和 encode 异常后 CPU、目标设备 + RNG state、dtype 及 image tiling state 都恢复；CPU 与每个声称支持的 accelerator 分支分别 + 覆盖，并加入重叠调用 fence。PR 中拟议的专用 VAE 单测按 review 被删除，目标 commit 没有 + 新增测试；PR body 的 focused/实机结果只能作为外部验证，不能冒充已提交回归覆盖。 + ^[PR #5703] + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 From 780ac2b75c8e81a0c8c767918b11b2011ad53ac4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 14:14:23 +0800 Subject: [PATCH 027/465] knowledge(vllm-omni): sync 556c5f88 (PR #5785) Upstream-Commit: 556c5f88a0f4b1d2e7285c6c69feecf0e3e39505 Upstream-PR: #5785 Knowledge-Delta: MiniMax H3 ComfyUI recipe provenance --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/minimax-h3/_index.md | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index eb848d1f7..5507bfa35 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7fb62db5c226558162dbee5b204d8d57479d28a0 + previous_audited_sha: 2eada79b99ae64acd835e5ca3794eb6ff60f4424 audited_ref: main - audited_sha: 2eada79b99ae64acd835e5ca3794eb6ff60f4424 + audited_sha: 556c5f88a0f4b1d2e7285c6c69feecf0e3e39505 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a1ecb2d21..0549a1acc 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 2eada79b99ae64acd835e5ca3794eb6ff60f4424`. The canonical machine baseline + 556c5f88a0f4b1d2e7285c6c69feecf0e3e39505`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c7c0cdac6..09c5a3524 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2eada79b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 556c5f88` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 45dfef0cd..c58c3ef24 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2eada79b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 556c5f88` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f158d4ef8..5ec7e8b91 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2eada79b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 556c5f88` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3bcac34de..005dd8869 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2eada79b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 556c5f88` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f7718013c..24723dae8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2eada79b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 556c5f88` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index aae20fbc7..0e0c983cd 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2eada79b` 验证存在 +- 源码校验：以上路径均已在 `main @ 556c5f88` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 70662ff3d..47c611f0f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2eada79b`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 556c5f88`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 7fb62db5 \ - --to 2eada79b \ + --from 2eada79b \ + --to 556c5f88 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 5edd3bfb9..db7634a2a 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5785", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- From c72f8e803aaf0f88aabb2550d02a73a4f56a7846 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 14:20:53 +0800 Subject: [PATCH 028/465] knowledge(vllm-omni): sync 589afcef (PR #5807) Upstream-Commit: 589afcef6c555c9b682fca1a3670a40fba507879 Upstream-PR: #5807 Knowledge-Delta: live ownership and governance routing --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/owners/_index.md | 9 +++++++-- 10 files changed, 19 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5507bfa35..20430b017 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2eada79b99ae64acd835e5ca3794eb6ff60f4424 + previous_audited_sha: 556c5f88a0f4b1d2e7285c6c69feecf0e3e39505 audited_ref: main - audited_sha: 556c5f88a0f4b1d2e7285c6c69feecf0e3e39505 + audited_sha: 589afcef6c555c9b682fca1a3670a40fba507879 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0549a1acc..5be2ed0b8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 556c5f88a0f4b1d2e7285c6c69feecf0e3e39505`. The canonical machine baseline + 589afcef6c555c9b682fca1a3670a40fba507879`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 09c5a3524..3d150baea 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 556c5f88` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 589afcef` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c58c3ef24..a90a284b2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 556c5f88` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 589afcef` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5ec7e8b91..22a724168 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 556c5f88` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 589afcef` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 005dd8869..211728e34 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 556c5f88` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 589afcef` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 24723dae8..2c068d433 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 556c5f88` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 589afcef` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0e0c983cd..5b86c18d3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 556c5f88` 验证存在 +- 源码校验：以上路径均已在 `main @ 589afcef` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 47c611f0f..3dbf7fb7e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 556c5f88`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 589afcef`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 2eada79b \ - --to 556c5f88 \ + --from 556c5f88 \ + --to 589afcef \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/owners/_index.md b/knowledge/repos/vllm-omni/owners/_index.md index c403743b0..aea866817 100644 --- a/knowledge/repos/vllm-omni/owners/_index.md +++ b/knowledge/repos/vllm-omni/owners/_index.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 负责人" created: 2026-08-11 -updated: 2026-08-11 +updated: 2026-09-02 type: index tags: [vllm-omni, models] -sources: [] +sources: [.github/CODEOWNERS, docs/community/governance.md] --- # vLLM-Omni 负责人 @@ -19,9 +19,14 @@ sources: [] - 模型代码入口、registry key 和别名；这些放 [`models/catalog.md`](../models/catalog.md)。 - 单个模型的实现与运行细节；这些放 `models/<模型>/`。 +- 上游完整 path owner 或 committer 人员快照；人员会变动，路径审查负责人以目标 pin 的 + `.github/CODEOWNERS` 为准，committer 身份与职责以 `docs/community/governance.md` 为准。 + 本目录只保留确有独立维护价值的模型路由，不把两份权威清单复制成易过期表格。 ## 目录内容 | 遇到什么 | 查看哪里 | 说明 | |---|---|---| +| 按 changed path 找 review owner | 上游 `.github/CODEOWNERS` | 权威且随 pin 变化；按 GitHub CODEOWNERS pattern 匹配，不从本地人员表推断 | +| 确认 committer 身份与公开职责 | 上游 `docs/community/governance.md` | governance roster 是角色权威来源，不等同于每条 path 的 required reviewer | | 关键模型与 Owner 清单 | [model_owner](model_owner.md) | 模型 → 负责人对照表 | From 872baf9d4219028a61856d6a37ffe675b2ddc754 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 14:28:54 +0800 Subject: [PATCH 029/465] knowledge(vllm-omni): sync 9f978923 (PR #5599) Upstream-Commit: 9f978923f2f537da6adac700f0d2330dc377ec33 Upstream-PR: #5599 Knowledge-Delta: consolidated diffusion execution modes and docs routing --- adapters/vllm_omni/release_baseline.yaml | 5 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/step-and-batching.md | 46 +++++++++++++++++-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/docs/design-doc-map.md | 8 ++-- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 11 files changed, 59 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 20430b017..cc5c418cb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 556c5f88a0f4b1d2e7285c6c69feecf0e3e39505 + previous_audited_sha: 589afcef6c555c9b682fca1a3670a40fba507879 audited_ref: main - audited_sha: 589afcef6c555c9b682fca1a3670a40fba507879 + audited_sha: 9f978923f2f537da6adac700f0d2330dc377ec33 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -43,6 +43,7 @@ path_owners: - .claude/skills/ - docs/ - examples/ + - mkdocs.yml - recipes/ model-executor: - vllm_omni/attention/ diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5be2ed0b8..55b5582c1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 589afcef6c555c9b682fca1a3670a40fba507879`. The canonical machine baseline + 9f978923f2f537da6adac700f0d2330dc377ec33`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3d150baea..efec5d249 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 589afcef` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9f978923` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a90a284b2..e142fbc6c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 589afcef` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9f978923` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md index 2f948346b..7d16e0697 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md +++ b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md @@ -1,16 +1,29 @@ --- title: "Step 执行合同与 batching 模式" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-09-02 type: guide tags: [vllm-omni, components, diffusion] -sources: [docs/design/feature/diffusion_step_execution.md, docs/design/feature/diffusion_request_level_batching.md, docs/design/feature/diffusion_continuous_batching.md] +sources: ["PR #5599", docs/design/feature/diffusion_continuous_batching.md, docs/user_guide/diffusion/execution_modes.md, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/sched/step_scheduler.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_step_pipeline.py] --- # Step 执行合同与 batching 模式 -官方 spec：`docs/design/feature/diffusion_{step_execution,request_level_batching,continuous_batching}.md` -（`main @ 5c390096` 复核）。 +官方设计入口已合并为 `docs/design/feature/diffusion_continuous_batching.md`，用户配置入口是 +`docs/user_guide/diffusion/execution_modes.md`（`main @ 9f978923` 复核）。旧的 step、request +batching 设计页及两份用户指南已删除；迁移依赖 MkDocs redirects，不能继续把旧路径当 source。 + +## 两个开关组成四种配置，不是四个 engine mode + +| `step_execution` | `max_num_seqs` | scheduler / 执行语义 | +|---|---:|---| +| false | 1 | `RequestScheduler`；一次完整 `forward()`，串行 request | +| false | >1 | `RequestScheduler`；兼容 request 融合为一次完整 `forward()` | +| true | 1 | `StepScheduler`；每个 tick 推进一个 request 的一个 denoise step | +| true | >1 | `StepScheduler`；兼容 request 共享 step wave，即 continuous batching | + +前两行在 engine 中同属 `REQUEST_BATCH`，后两行同属 `STEP_BATCH`。`max_num_seqs>1` 只是容量， +不会自动赋予 pipeline 批处理能力。 ## step_execution 不是通用开关 @@ -31,6 +44,31 @@ sources: [docs/design/feature/diffusion_step_execution.md, docs/design/feature/d **不保证单请求延迟收益**。基础 step 合同不变，改动集中在 scheduler 与 runner 层 （`vllm_omni/diffusion/sched/`、`worker/`）。 +request mode 的 `max_num_seqs>1` 要求 pipeline 显式声明 `supports_request_batch=True`，否则 +engine 初始化失败。`request_batch_max_wait_ms` 只在 request mode、支持 request batch、当前 +没有 running request 且等待值大于零时帮助突发请求成批；它用首请求延迟换 batch formation。 + +## step capability 必须验证 grouped contract + +`supports_step_execution=True` 只证明四段 stateful contract（`prepare_encode`、 +`denoise_step(input_batch, *, states=...)`、`step_scheduler`、`post_decode`）存在，不证明 +`denoise_step` 能处理多个 state。Qwen-Image 支持 grouped step；HunyuanImage3 只有 resolved +self-attention backend 为 `TORCH_SDPA` 时支持 grouped step；Helios 会拒绝 `len(states) != 1`， +因此必须使用 `--step-execution --max-num-seqs 1`。新增 pipeline 不能只凭 capability flag +宣称 continuous batching，须用两个重叠且兼容的真实请求覆盖 state 隔离、abort 与输出归属。 + +step mode 当前拒绝所有 diffusion cache backend，但这不等于拒绝 inter-stage KV transfer： +runner 对新 admission 先调用 `receive_multi_kv_cache_distributed()`，再 `prepare_encode()`，且有 +顺序测试。限制描述必须区分这两个机制。 + +## output stream 是统一生命周期 + +streaming 与 non-streaming caller 都消费 `DiffusionEngine.step_streaming()` 对应的 per-request +queue；前者转发每个 chunk，后者 drain 后只返回最终结果。terminal scheduler result 负责完成 +request state；consumer cancel/disconnect 删除 delivery queue，但 scheduler 仍须完成清理； +shutdown 给剩余 stream 发送 error。兼容 wrapper `step()` 与 +`async_add_req_and_wait_for_response()` 已 deprecated，新集成应使用 streaming API。 + ## 相关 - 噪声调度/采样归 [Diffusion 组件](_index.md)；请求级排队语义见 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 22a724168..f94beaf7e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 589afcef` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9f978923` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 211728e34..54c8ab291 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 589afcef` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9f978923` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2c068d433..7688e762e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 589afcef` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9f978923` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5b86c18d3..d8d85dcb7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 589afcef` 验证存在 +- 源码校验：以上路径均已在 `main @ 9f978923` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index 8823cd029..6ed114fd8 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -4,13 +4,13 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, docs] -sources: [docs/design/index.md, docs/design/architecture_overview.md, docs/design/feature/omni_async_output_materialization.md] +sources: [docs/design/index.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/omni_async_output_materialization.md, docs/user_guide/diffusion/execution_modes.md] --- # 官方设计文档地图（docs/design/**） -`main @ d5d47e9b` 复核。注意：官方 `docs/design/index.md` 只列了子集（Architecture -Overview、6 篇 feature、metrics、3 篇 module）；**完整树比索引大得多**—— +`main @ 9f978923` 复核。注意：官方 `docs/design/index.md` 只列了子集（Architecture +Overview、4 篇 feature、metrics、3 篇 module）；**完整树比索引大得多**—— `feature/` 下还有 async_chunk、cache_dit、teacache、prefix_caching、7 篇并行策略、 `omni_connectors/` 逐后端 spec 等未入索引的文档，找 spec 时直接 `ls docs/design/feature/`。 @@ -23,7 +23,7 @@ Overview、6 篇 feature、metrics、3 篇 module）；**完整树比索引大 | `module/dit_module.md` | Diffusion 引擎/调度/worker/pipeline/加速组件 | [diffusion](../components/diffusion/architecture.md) | | `module/entrypoint_module.md` | **stub（"update soon"）——上游文档缺口** | [serving](../components/serving/_index.md) | | `feature/disaggregated_inference.md` + `omni_connectors/*` | connector 选择矩阵与逐后端 spec | [distributed](../components/distributed/connector-backends.md) | -| `feature/diffusion_step_execution / request_level / continuous_batching` | step 合同与两种 batching | [diffusion](../components/diffusion/step-and-batching.md) | +| `feature/diffusion_continuous_batching.md` + `user_guide/diffusion/execution_modes.md` | 合并后的开发者设计与用户配置入口：request/step 执行、两种 batching、统一 output stream | [diffusion](../components/diffusion/step-and-batching.md) | | `feature/async_chunk.md` | 跨 stage 分块流式 | [distributed](../components/distributed/async-chunk.md) | | `feature/cache_dit.md`、`teacache.md`、`prefix_caching.md` | 缓存加速 | [diffusion](../components/diffusion/cache-acceleration.md) | | `feature/{tensor,pipeline,sequence,expert,cfg,vae}_parallel.md`、`hsdp.md` | 并行策略 | [diffusion](../components/diffusion/parallelism.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3dbf7fb7e..2b6bec98c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 589afcef`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9f978923`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 556c5f88 \ - --to 589afcef \ + --from 589afcef \ + --to 9f978923 \ --repo \ --mode report-only ``` From f85e08347518c75be984ae596c34b4648b42a748 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 14:36:55 +0800 Subject: [PATCH 030/465] knowledge(vllm-omni): sync b4581b29 (PR #5634) Upstream-Commit: b4581b293beb837a2936b16c0cad6d2834fed9f6 Upstream-PR: #5634 Knowledge-Delta: Cosmos3 Nano MI350X evidence boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/cosmos3/_index.md | 11 ++++++++-- .../repos/vllm-omni/models/cosmos3/rules.md | 21 +++++++++++++++++-- 11 files changed, 40 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cc5c418cb..352f740ea 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 589afcef6c555c9b682fca1a3670a40fba507879 + previous_audited_sha: 9f978923f2f537da6adac700f0d2330dc377ec33 audited_ref: main - audited_sha: 9f978923f2f537da6adac700f0d2330dc377ec33 + audited_sha: b4581b293beb837a2936b16c0cad6d2834fed9f6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 55b5582c1..fc55b2c39 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 9f978923f2f537da6adac700f0d2330dc377ec33`. The canonical machine baseline + b4581b293beb837a2936b16c0cad6d2834fed9f6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index efec5d249..5406596a7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9f978923` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b4581b29` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e142fbc6c..dd95a46b6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9f978923` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b4581b29` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f94beaf7e..3ba557f15 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9f978923` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b4581b29` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 54c8ab291..bf86c895c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9f978923` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b4581b29` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7688e762e..f15124af7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9f978923` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b4581b29` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d8d85dcb7..71b486d5d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9f978923` 验证存在 +- 源码校验：以上路径均已在 `main @ b4581b29` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2b6bec98c..5e2c6e02b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9f978923`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b4581b29`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 589afcef \ - --to 9f978923 \ + --from 9f978923 \ + --to b4581b29 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/_index.md b/knowledge/repos/vllm-omni/models/cosmos3/_index.md index e4a3a3374..feb68425e 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/_index.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/_index.md @@ -1,10 +1,10 @@ --- title: "Cosmos3" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5001", vllm_omni/diffusion/models/cosmos3/] +sources: ["PR #5001", "PR #5634", recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/platforms/rocm/platform.py] confidence: high --- @@ -31,6 +31,13 @@ transformer 还要维护自己的 layerwise-offload block 声明。architecture 变体专有不变量与描述直达源码入口见 [rules](rules.md#direct-代码快速入口)。新模型通用验证见 [model validation](../../review/guides/model-validation.md)。 +## ROCm evidence scope + +ROCm recipe 的 MI350X latency/显存只来自 `b3f4fbf9` 上单卡 gfx950、Nano T2I/T2V、 +guardrails off 的一次 warmup + 一次测量，不是当前 pin 或通用 ROCm 保证。gfx942 只命中 +platform 的 AITER capability gate，未在该 PR 实测；质量、multi-GPU 与 Cosmos3-Super 也未覆盖。 +引用具体数字或扩展支持矩阵时按本页审查入口中的 `COSMOS-3b` 复核。 + ## 什么时候查这里 - 审查 Cosmos3 Edge/Distilled scheduler、RNG、guidance、offload 或 capability claim。 diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index c8371aa95..c6ecb473a 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -1,10 +1,10 @@ --- title: "Cosmos3 规则" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5001"] +sources: ["PR #5001", "PR #5634", recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/platforms/rocm/platform.py] confidence: high --- @@ -21,6 +21,7 @@ confidence: high | seed/generator、逐步噪声 | COSMOS-2a | `pipeline_cosmos3.py::Cosmos3OmniDiffusersPipeline` 的 request-local sampling 路径 | | `guidance=0`、request extra | COSMOS-2b | `model_extras/cosmos3.py` → pipeline sampling params | | online/offload/HSDP/VAE parallel 支持声明 | COSMOS-3a | capability 文档/recipe → 对应公开入口与实现路径 | +| ROCm、MI350X、AITER、latency/peak memory | COSMOS-3b | recipe 的 measurement commit/protocol → `platforms/rocm/platform.py` backend gate → 目标配置复测 | 若描述只写 Cosmos3，先从 registry 的 class key 进入 pipeline；只有命中共享 RNG、graph 或 offload 机制时再加 [Diffusion owner](../../components/diffusion/rules.md)。 @@ -65,5 +66,21 @@ confidence: high - 禁止：用 offline unit test 支撑 online/HSDP/offload 多项 claim。 - 验收：公开矩阵逐项引用当前 head 证据；pending 项明确未验证或暂不声明。 ^[PR #5001] +## COSMOS-3b — 硬件 recipe 数字必须绑定测量 commit、协议和未覆盖维度 + +- 触发：引用 Cosmos3 ROCm latency、显存、determinism、backend 或最低硬件声明。 +- 强制：PR #5634 的数值只绑定 vLLM-Omni `b3f4fbf9`、单张 MI350X/gfx950、HIP 7.2、 + guardrails off、Nano T2I/T2V、每配置一次 warmup 加一次 measured run。T2I 1024²/50 steps + 报告约 2.7 s、49.5 GiB reserved；T2V 1280×720/189 frames/35 steps 报告 161 s、 + 120 GiB reserved/95 GiB allocated；tiling、layerwise offload、FP8 分别报告约 + 183 s/38/36 GiB、163 s/84/69 GiB、149 s/107/82 GiB。数字是该协议的观察值， + 不是 merge target `b4581b29` 的复测结果。 +- 禁止：把单次 measured run 变成稳定性能界、从 38 GiB peak 推断已验证 40 GiB 可运行， + 或把同配置 byte-identical 外推为跨 flags/hardware determinism。ROCm platform 的 AITER gate + 包含 gfx942/gfx950，但 PR 只实测 gfx950；gfx942、multi-GPU、Cosmos3-Super 均不得标已验证。 +- 验收：任何 target-head 或硬件矩阵 claim 都重新记录 commit、完整软件栈、warm/cold、样本数、 + reserved/allocated 定义、backend assertion 与输出检查；quality 必须有独立 metric/golden。 + PR #5634 只验证尺寸、帧数和同配置复现，明确没有 quality evaluation。^[PR #5634] + 共享 RNG/graph 规则见 [Diffusion rules](../../components/diffusion/rules.md)；公开证据分层见 [model adaptation guardrails](../../review/guides/model-adaptation-guardrails.md)。 From d3bae935c81f53469eed4bc8598afb24bb394be9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 15:02:16 +0800 Subject: [PATCH 031/465] knowledge(vllm-omni): sync 1c2a81f6 (PR #5764) Upstream-Commit: 1c2a81f6d84aea4fff53bd2f894c2a287c237245 Upstream-PR: #5764 Knowledge-Delta: MiniMax H3 DLO residency, staging, and consumer-GPU evidence --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 12 +++++- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../vllm-omni/models/minimax-h3/_index.md | 5 ++- .../vllm-omni/models/minimax-h3/rules.md | 38 ++++++++++++++++++- 12 files changed, 63 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 352f740ea..8cebc4ada 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9f978923f2f537da6adac700f0d2330dc377ec33 + previous_audited_sha: b4581b293beb837a2936b16c0cad6d2834fed9f6 audited_ref: main - audited_sha: b4581b293beb837a2936b16c0cad6d2834fed9f6 + audited_sha: 1c2a81f6d84aea4fff53bd2f894c2a287c237245 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fc55b2c39..05f037e83 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - b4581b293beb837a2936b16c0cad6d2834fed9f6`. The canonical machine baseline + 1c2a81f6d84aea4fff53bd2f894c2a287c237245`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5406596a7..ddcd0b932 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b4581b29` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1c2a81f6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index dd95a46b6..7f9f3be6f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b4581b29` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1c2a81f6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 6cc162b7a..885f573a7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5737", "PR #5802", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/, tests/diffusion/offloader/test_distributed_layerwise_backend.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5737", "PR #5764", "PR #5802", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/, tests/diffusion/offloader/test_distributed_layerwise_backend.py] confidence: high --- @@ -141,11 +141,17 @@ confidence: high - 触发：新增 `enable_distributed_layerwise_offload`、`dlo_use_allgather`、模型 `OffloadPlan` 或 block discovery。 - 强制：明确 rank 本地 CPU shard、固定双 GPU buffer、H2D/AllGather stream 和模型 - block list 的 owner；`dlo_use_allgather=false` 走每 rank full-weight 路径，开启 + block list 的 owner；`dlo_use_allgather=false` 保留 standard loader 已生成的每-rank + ready-to-run layout（没有其他分片维度时才是 full weight，TP 下则是 TP-local shard），开启 AllGather 时不得再叠加已 DTensor-sharded 的 HSDP 参数。共享 output buffer 可按所有 hook 的最大 block 分配，但每个 hook 调 collective 前必须按 dtype 截成自己的 `_ag_output_sizes[dtype] == dp_size * local_shard.numel()` 前缀；block-flat-relative repoint offset 只能在这段实际 block buffer 内解释。 +- 强制：非 DiT component 的 staging 必须由 `OffloadPlan` 显式声明：encoder block group + rank-local streaming 不借用 DiT AllGather group，on-demand component 由 pipeline 在真实 + encode/decode stage 成对 load/offload，resident layer 只对声明的 DiT path 生效。no-AllGather + 路径保留 standard loader 产生的 TP-local layout；若模型 loader 还做 QKV/MLP transform， + mmap 必须显式 opt out，直到 transform-before-shard 等价性有测试。 - 禁止：用 heuristic 找不到 block 时静默假装 offload 已启用；对 HSDP 参数二次分片； 把 CPU 内存、AllGather 同步和设备显存开销隐藏在一个泛化的 `enable_cpu_offload` 开关； 把 max-sized shared buffer 原样交给较小 block 的 `all_gather_into_tensor`。 @@ -154,6 +160,8 @@ confidence: high 不同 block size 验证 small/large hook 都满足 collective size equality、重建权重和 repoint。 当前回归用 mocked collective 检查 8/32-element、dp=2 合同；没有真实 multi-rank collective 覆盖，PR 的 8×NPU H3 E2E 说明也缺少已填写的 commit 证据，不能升级为验证结论。 ^[PR #5802] + DP concurrent wave 只可在 DLO+AllGather 开启，并从 `sampling_params.extra_args` 比较 canonical + signature；no-AllGather 的 TP-local request 不得误走 fused batch。^[PR #5764] ## 质量阈值与资源辅助 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3ba557f15..e03c84d7b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b4581b29` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1c2a81f6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index bf86c895c..fc484d439 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b4581b29` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1c2a81f6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f15124af7..7c6d04053 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b4581b29` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1c2a81f6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 71b486d5d..b7a82d655 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b4581b29` 验证存在 +- 源码校验：以上路径均已在 `main @ 1c2a81f6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5e2c6e02b..aa3520a34 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b4581b29`（2026-08-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1c2a81f6`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 9f978923 \ - --to b4581b29 \ + --from b4581b29 \ + --to 1c2a81f6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index db7634a2a..b1527d483 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5785", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -44,6 +44,9 @@ confidence: high - conditioned image/video VAE 用固定内部 seed，并在 `fork_rng` 中播种 CPU 与参数所在设备后 恢复 state；MUSA recipe 记录 MTT S5000 验证。目标实现实际接纳所有非 CPU device，而实机 RNG 证据只覆盖 CUDA/MUSA；支持边界与并发缺口见 MMH3-2c。 +- 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention + 和 eager execution；resident layers 只改变 HBM/transfer，不减少 pinned host master。 + 实现合同与 standalone-audio staging 缺口见 MMH3-3a，5090/4090 证据边界见 MMH3-3b。 ## ComfyUI 请求路由 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 02cf0e98c..4310557a4 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5737", "PR #5752", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] +sources: ["PR #5703", "PR #5737", "PR #5752", "PR #5764", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] confidence: high --- @@ -22,6 +22,8 @@ confidence: high | FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | +| DLO、TP-local、resident layers、encoder/VAE staging | `MMH3-3a` | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | +| RTX 5090/4090、24/32 GiB、consumer profile | `MMH3-3b` | recipe measurement commit/run record → exact target topology → quality/capacity validation | ## MMH3-1a — component namespace 与 checkpoint transform 必须在 active loader 前闭合 @@ -121,5 +123,39 @@ confidence: high 新增测试；PR body 的 focused/实机结果只能作为外部验证，不能冒充已提交回归覆盖。 ^[PR #5703] +## MMH3-3a — H3 DLO 必须保持 loader layout 与 component stage 配对 + +- 触发：H3 `OffloadPlan`、`dlo_no_use_allgather`、resident layer、text encoder/VAE staging + 或 DLO 与 CPU/layerwise offload 组合。 +- 强制：H3 先走 regular loader 形成 TP-local grouped-QKV/fused-MLP layout，再以 no-AllGather + H2D streaming 使用它；`_supports_mmap_loading=False` 是安全门，不能绕过。plan 将 + `token_refiner.blocks` 独立 stream，text encoder 的 vision/text block rank-local stream， + video/audio VAE 按 encode/decode stage on-demand，leading `transformer` blocks 只在 denoise + stage resident 并在 decode 前释放。`dlo_resident_layers>0` 必须与 no-AllGather 配对。 +- 禁止：让 encoder TP shard 进入 DiT AllGather；把 DLO+CPU offload 的早期分支优先级交给 + generic CPU-offload hook；只因 component 有 `offload_to_cpu` 就推断每个 caller 都正确 staging。 + 目标 pin 的 `_encode_audio_conditions(standalone_audios)` 没有进入 `_component_on_device`， + 会在前一个 embedded-audio context 已关闭后直接使用 CPU-bound audio VAE；没有外层 stage + context 或专属测试证明 image+standalone-audio Ref2VA 的 DLO device/performance 合同。 +- 验收：分别覆盖 text、image、video、embedded audio、standalone audio 与 decode 的 + load→use→finally-offload 次序，异常也释放；regular-loader transform sentinel 到 TP-local stream + 后仍正确；resident layers 只围住 denoise。补齐上述 standalone-audio gap 前，不能用其他 + component staging 单测宣称所有 Ref2VA 输入已覆盖。^[PR #5764] + +## MMH3-3b — consumer-GPU profile 是有边界的容量证据 + +- 触发：引用 H3 2×RTX 5090/4090 可运行性、延迟、峰值或 resident-layer 默认值。 +- 强制：5090 证据只绑定 vLLM-Omni `ae6577ea` 的一次 2×32 GiB、T2VA、1344×768、 + 124 frames、50 steps run：8m38s，约 22.6 GiB/GPU 是 sampled `nvidia-smi`，不是 allocator + high-water。4090 的 2×24 GiB/1024×576/12 resident layers 只来自 2×B300 的 5-step + capacity proxy；它不是 RTX 4090 latency 或 full-run validation。 +- 禁止：把单次未 warmed 的 5090 run 写成 benchmark，把 B300 proxy 写成 target hardware， + 或从 HBM fit 推断 host fit。每个 FL2VA/Ref2VA partition 约 135 GiB，no-AllGather worker 的 + pinned CPU master 不因 resident layers 增加而消失；recipe 要求至少 200 GiB available RAM。 +- 验收：目标硬件用单一 partition、exact TP2/no-AllGather/VAE PP2/cuDNN/eager topology 复测， + 分别记录 allocator peak、重复 latency 和 joint video/audio quality。目标 pin 引用的 + `examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh` 实际不存在，因此不能把其 + four-task/MP4 validation 描述成可执行入口；补文件或改文档后再验收。^[PR #5764] + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 From b09c80abbe0dbeab511414690436d5227e45e16e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 15:12:11 +0800 Subject: [PATCH 032/465] knowledge(vllm-omni): sync 9235b0ae (PR #5382) Upstream-Commit: 9235b0ae651423bd1b53dac491b564b8afdf8c57 Upstream-PR: #5382 Knowledge-Delta: MiniCPM-o Whisper chunk-mask vectorization contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/minicpm-o-4-5/_index.md | 6 +++++- .../vllm-omni/models/minicpm-o-4-5/rules.md | 19 ++++++++++++++++++- 11 files changed, 35 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8cebc4ada..d86da4310 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b4581b293beb837a2936b16c0cad6d2834fed9f6 + previous_audited_sha: 1c2a81f6d84aea4fff53bd2f894c2a287c237245 audited_ref: main - audited_sha: 1c2a81f6d84aea4fff53bd2f894c2a287c237245 + audited_sha: 9235b0ae651423bd1b53dac491b564b8afdf8c57 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 05f037e83..49abf5286 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 1c2a81f6d84aea4fff53bd2f894c2a287c237245`. The canonical machine baseline + 9235b0ae651423bd1b53dac491b564b8afdf8c57`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ddcd0b932..464749189 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1c2a81f6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9235b0ae` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7f9f3be6f..7dd11b653 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1c2a81f6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9235b0ae` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e03c84d7b..220ca618a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1c2a81f6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9235b0ae` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index fc484d439..dc8b20e87 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1c2a81f6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9235b0ae` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7c6d04053..278c3f68a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1c2a81f6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9235b0ae` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b7a82d655..fd10063d4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1c2a81f6` 验证存在 +- 源码校验：以上路径均已在 `main @ 9235b0ae` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index aa3520a34..c630393c8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1c2a81f6`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9235b0ae`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from b4581b29 \ - --to 1c2a81f6 \ + --from 1c2a81f6 \ + --to 9235b0ae \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 6ae44a275..38e5daf04 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5638", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/] +sources: ["PR #3642", "PR #5382", "PR #5638", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/] confidence: high --- @@ -31,6 +31,10 @@ Code2Wav 在所有平台使用树内 `MiniCPMO45Token2wav`。CUDA 可显式开 Campplus TensorRT；这是可关闭的局部加速，不替换 encoder/HiFT，也不会自动启用 Step-Audio2 的 token2wav。engine cache/profile 与 fallback 门禁见 MCPMO-1c。 +Thinker 的 Whisper/APM audio encoder 仍构造 dense `[B,1,T,T]` mask；chunk mask 已用 +broadcasted query/key index 代替逐 row Python fill，但不改变 chunk/left-context/lookahead +边界，也不消除 O(T²) storage。语义与证据门禁见 MCPMO-2b。 + 描述直达源码与模型专有门禁见 [rules](rules.md#direct-代码快速入口)；新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 47ef2ee14..8636f3b3c 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5638", "PR #6154", "PR #6170", "PR #6318"] +sources: ["PR #3642", "PR #5382", "PR #5638", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py] confidence: high --- @@ -17,6 +17,7 @@ confidence: high | PR 描述信号 | 规则组 | 第一批源码 | |---|---|---| | MiniCPM-o 4.5、`minicpmo_4_5`、版本识别 | MCPMO-2a | `config/pipeline_registry.py::OMNI_PIPELINES["minicpmo_4_5"]` → `model_executor/models/minicpmo_4_5/pipeline.py`；`model_executor/models/registry.py::_OMNI_MODELS` | +| Whisper/APM、chunk attention mask、left context/lookahead | MCPMO-2b | `minicpmo_4_5_omni_llm.py::{get_audio_hidden_states,subsequent_chunk_mask}` → audio encoder；row-by-row oracle test | | tokenizer/processor、`trust_remote_code` | MCPMO-1a | pipeline/config factory → `model_executor/models/minicpmo_4_5/` loader | | TTS extra、backend 初始化、空音频 | MCPMO-1b | `minicpmo_4_5_omni_tts.py::MiniCPMO45OmniTTSForConditionalGeneration`、`minicpmo_4_5_token2wav.py::MiniCPMO45Token2wav` | | Code2Wav TensorRT、DiT/Campplus、engine cache/profile | MCPMO-1c | `minicpmo_4_5_code2wav.py::MiniCPMO45Code2Wav` → `batched_token2wav.py::BatchedToken2Wav._estimator_step`；共享实现 `step_audio2/step_audio2_dit_trt.py` | @@ -68,6 +69,22 @@ confidence: high - 禁止：仅用 architecture 集合相交。 - 验收：4.5、旧版和无版本字段 fixture 分别命中 4.5、旧 pipeline 或明确失败。 ^[PR #3642] +## MCPMO-2b — audio chunk mask 向量化必须保持逐行边界语义 + +- 触发：修改 Whisper/APM `subsequent_chunk_mask`、chunk size、left-context 或 lookahead。 +- 强制：对每个 query row，end 保持 `min((row // chunk_size + 1) * chunk_size + + num_lookhead, size)`；`num_left_chunks < 0` 从 key 0 开始，否则 start 保持 + `max((row // chunk_size - num_left_chunks) * chunk_size, 0)`。可用 query/key index + broadcast 一次构造 dense bool `[T,T]` mask，但不得改变 caller 与 padding mask 的 OR/取反语义。 +- 禁止：把移除 O(T) Python row loop 描述成移除 O(T²) mask storage/attention；把 reviewer + 建议的 `@staticmethod` 当成已合并（目标仍保留 instance-method interface）；或声称所有 TTFT + case 都改善。PR 自己记录短输入/较大 batch 的 scheduler noise 会淹没局部节省。 +- 验收：row-by-row oracle 覆盖 size 0、chunk 边界两侧、无限/有限/零 left context 与 lookahead， + 并在真实 caller 组合 padding mask。目标提交的 CPU 参数化测试覆盖 216 组纯 mask parity， + 但不覆盖 CUDA kernel 数、完整 audio embedding 或 serving。PR 的 CUDA timing/bitwise embedding + 证据来自 `ee33954dff27da317be597449a6c1b5a5df4052b`，不是 merge target `9235b0ae` 的复测， + 因而只能作为外部局部性能证据。^[PR #5382] + ## MCPMO-3a — TTS stage 的 batch 能力与 runtime_info 消费一致 - 触发：talker/TTS stage 的 `max_num_seqs`、batching 或 runtime bridge。 From 0d8c293b4904099b25a462ba371d4892bea80a98 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 15:27:31 +0800 Subject: [PATCH 033/465] knowledge(vllm-omni): sync 240e6655 (PR #5829) Upstream-Commit: 240e6655be1ded9a5635cf98638f325188ce3267 Upstream-PR: #5829 Knowledge-Delta: MiniMax H3 FP8 quality fixture input contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/minimax-h3/_index.md | 2 +- .../vllm-omni/models/minimax-h3/rules.md | 19 ++++++++++++------- 11 files changed, 25 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d86da4310..6926bc537 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1c2a81f6d84aea4fff53bd2f894c2a287c237245 + previous_audited_sha: 9235b0ae651423bd1b53dac491b564b8afdf8c57 audited_ref: main - audited_sha: 9235b0ae651423bd1b53dac491b564b8afdf8c57 + audited_sha: 240e6655be1ded9a5635cf98638f325188ce3267 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 49abf5286..d6cb22128 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 9235b0ae651423bd1b53dac491b564b8afdf8c57`. The canonical machine baseline + 240e6655be1ded9a5635cf98638f325188ce3267`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 464749189..00702705c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9235b0ae` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 240e6655` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7dd11b653..ea63a0fd4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9235b0ae` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 240e6655` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 220ca618a..00c28ec9a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9235b0ae` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 240e6655` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index dc8b20e87..56914a24a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9235b0ae` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 240e6655` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 278c3f68a..3ea520b62 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9235b0ae` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 240e6655` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fd10063d4..1eb9507a2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9235b0ae` 验证存在 +- 源码校验：以上路径均已在 `main @ 240e6655` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c630393c8..0dae279d7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9235b0ae`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 240e6655`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 1c2a81f6 \ - --to 9235b0ae \ + --from 9235b0ae \ + --to 240e6655 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index b1527d483..8c2316876 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", "PR #5829", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 4310557a4..86ae467dc 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5737", "PR #5752", "PR #5764", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] +sources: ["PR #5703", "PR #5737", "PR #5752", "PR #5764", "PR #5829", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] confidence: high --- @@ -47,13 +47,17 @@ confidence: high - 触发：改变 H3 FP8 layer scope、kernel/loader、quality threshold、offload 或 nightly lane。 - 强制：H3 online FP8 与 layerwise offload 当前标为 incompatible；resident FP8 可与 TP、VAE tiling 组合。BF16/FP8 A/B 使用同一 immutable checkpoint、prompt、seed、size、frames、 - steps 与并行/资源设置；joint output 同时检查 video LPIPS 和 32 kHz audio 的 spectral - cosine/RMS ratio。PSNR、MAE 与 peak memory 是 report-only 指标，不能冒充 gate。 + steps、输入合同与并行/资源设置；T2VA case 即使显式给出 width/height，也必须在 + `sampling_params.extra_args` 带合法 named `aspect_ratio`。joint output 同时检查 video LPIPS + 和 32 kHz audio 的 spectral cosine/RMS ratio。PSNR、MAE 与 peak memory 是 report-only + 指标，不能冒充 gate。 - 禁止：只验视频就宣称 H3 joint-output quality；把一次 22% memory observation 写成稳定 上界；让 FP8+layerwise offload 到 Cutlass kernel 才因 flattened-weight stride 失败。 - 验收：当前 2x H100 case 的 gates 是 LPIPS <= 0.20、audio spectral cosine >= 0.80、 RMS ratio 在 0.50–2.00，且 sample rate 为 32000；单测还要证明 phase-tolerant metric - 接受相移但拒绝 spectral drift。 ^[PR #5737] + 接受相移但拒绝 spectral drift。质量 test 的 T2VA `aspect_ratio="16:9"` 是同一 case 的 + 必填输入，不是量化变量；输入合同演进后应修正 fixture，不能靠跳过 case 或放宽质量 gate + 恢复 CI。^[PR #5737] ^[PR #5829] ## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 @@ -64,9 +68,10 @@ confidence: high image/video visual reference；image≤9、video≤3、standalone audio≤3、总数≤12，允许 image-only 及 image/video/audio mixed matrix，但拒绝 audio-only。 - 强制：输出固定 24 FPS、32 kHz audio、duration 4–15 秒。省略 width/height 时 - `short_edge` 只能为 768；T2VA 必须显式选择 `21:9`、`16:9`、`4:3`、`1:1`、`3:4`、 - `9:16`，FL2VA 跟随首图 geometry 并忽略通用 ratio override，Ref2VA 默认 `16:9`， - `adaptive`/`auto` 是该默认的 alias。 + `short_edge` 只能为 768；T2VA 即使同时给出 width/height，仍必须显式选择 `21:9`、 + `16:9`、`4:3`、`1:1`、`3:4`、`9:16`，因为 named ratio 在 explicit-dimension branch + 之前解析。FL2VA 跟随首图 geometry 并忽略通用 ratio override，Ref2VA 默认 `16:9`， + `adaptive`/`auto` 是该默认的 alias。^[PR #5829] - 强制：`num_outputs_per_prompt` 仅为 1–10，各输出 seed 是 `seed + output_index`；pipeline tensor output 与未被公开 caller 使用的 `generate_videos()` 保留 fan-out。此 pin 上 public async job 与 `/v1/videos/sync` 都调用 `generate_video_bytes()`，多输出时告警并只持久化/返回 From 8ecb7735fe051ed550807c1ee377563c1b5b9535 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 15:44:08 +0800 Subject: [PATCH 034/465] knowledge(vllm-omni): sync f2e3d2e2 (PR #5801) Upstream-Commit: f2e3d2e25bebc0c7110ccb945d7dcf4be81bf6db Upstream-PR: #5801 Knowledge-Delta: MiniMax H3 fused RMSNorm and RoPE contracts --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 21 +++++++++++++++---- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../vllm-omni/models/minimax-h3/_index.md | 4 +++- .../vllm-omni/models/minimax-h3/rules.md | 19 ++++++++++++++++- 12 files changed, 49 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6926bc537..c23683fc2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9235b0ae651423bd1b53dac491b564b8afdf8c57 + previous_audited_sha: 240e6655be1ded9a5635cf98638f325188ce3267 audited_ref: main - audited_sha: 240e6655be1ded9a5635cf98638f325188ce3267 + audited_sha: f2e3d2e25bebc0c7110ccb945d7dcf4be81bf6db # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d6cb22128..ad51b2279 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 240e6655be1ded9a5635cf98638f325188ce3267`. The canonical machine baseline + f2e3d2e25bebc0c7110ccb945d7dcf4be81bf6db`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 00702705c..508998ab4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 240e6655` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f2e3d2e2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ea63a0fd4..b738f4c99 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 240e6655` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f2e3d2e2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 885f573a7..efebb4694 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5737", "PR #5764", "PR #5802", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/, tests/diffusion/offloader/test_distributed_layerwise_backend.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] confidence: high --- @@ -20,7 +20,7 @@ confidence: high | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| -| CUDA Graph、compile、fused solver、eager parity、tensor dtype/device、async output | `execution-parity`：`DIFF-1a`–`1d` | `compile.py::regionally_compile` → runner batch → model denoise/solver/output pump | +| CUDA Graph、compile、fused solver/norm/RoPE、eager parity、tensor dtype/device、async output | `execution-parity`：`DIFF-1a`–`1e` | `compile.py::regionally_compile` → shared layer/backend → model denoise/solver/output pump | | seed、request-local generator、guidance=0、并发 RNG、batched generators | `execution-parity`：`DIFF-1b` | `inputs/data.py::OmniDiffusionSamplingParams` → runner `_initialize_generator` → request-batch generator collate | | ModelOpt/checkpoint adapter、weight/scale remap、unknown tensor、resolution path | `checkpoint-distributed`：`DIFF-2a` | `diffusers_loader.py::{_get_checkpoint_adapter,load_weights}` → `modelopt.py::{_resolve_target_and_output_names,adapt}` | | host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | @@ -33,8 +33,8 @@ confidence: high | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| -| `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d` | -| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、async output readiness | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d` | +| `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | +| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、async output readiness | `DIFF-1a`–`1e` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload | `DIFF-2a`, `DIFF-2b`, `DIFF-2c`, `DIFF-2d`, `DIFF-2e` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4f` | @@ -84,6 +84,19 @@ confidence: high `tests/diffusion/test_async_output_worker.py`、`test_result_pump.py` 和 `tests/diffusion/test_ipc_async.py`。 +### DIFF-1e — fused layer 必须保留精度、布局与平台分派合同 + +- 触发：共享 norm/RoPE 增加 fused op、平台专用实现、compile 分支或可配置 parameter dtype。 +- 强制：把 parameter 存储 dtype 与 reduction/累加 dtype 分开约束；CUDA/HIP eager 可尝试 + fused op，但异常必须回退到数值等价的 native 实现，`torch.compile` tracing 直接走 native， + NPU/MindIE 等专用 backend 则按其真实参数与频率布局调用。RoPE 适配器必须显式区分 full-dim + tiled frequency 与 fused kernel 的 half layout，并保持未旋转 head tail 不变。 +- 禁止：用 platform dispatch 推断所有 backend 接收同一布局；用 CPU/reference 或 mock 参数 + wiring 测试声称真实 CUDA/HIP/NPU fused kernel 已完成数值 parity。 +- 验收：native reference 覆盖 parameter dtype、FP32 accumulation、旋转维度与 untouched tail； + compile 路径断言不调用 fused op，平台 mock 只作为 wiring 证据。声称 fused parity 还需目标 + 硬件上的同输入数值测试。^[PR #5801] + ## Checkpoint 与分布式加载 ### DIFF-2a — checkpoint remap 必须追到已注册且真实消费的目标 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 00c28ec9a..e5ffd6f3b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 240e6655` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f2e3d2e2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 56914a24a..0719070ec 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 240e6655` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f2e3d2e2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3ea520b62..84eb74ebd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 240e6655` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f2e3d2e2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1eb9507a2..1049eabb5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 240e6655` 验证存在 +- 源码校验：以上路径均已在 `main @ f2e3d2e2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0dae279d7..d391d86c3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 240e6655`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f2e3d2e2`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 240e6655 \ + --to f2e3d2e2 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 8c2316876..4e1330993 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", "PR #5829", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", "PR #5801", "PR #5829", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -44,6 +44,8 @@ confidence: high - conditioned image/video VAE 用固定内部 seed，并在 `fork_rng` 中播种 CPU 与参数所在设备后 恢复 state；MUSA recipe 记录 MTT S5000 验证。目标实现实际接纳所有非 CPU device，而实机 RNG 证据只覆盖 CUDA/MUSA；支持边界与并发缺口见 MMH3-2c。 +- H3 q/k 使用共享 RMSNorm（BF16 gamma、native FP32 accumulation）与 NeoX RoPE；每个 + 128 维 head 只旋转前 96 维并保留后 32 维，平台布局和验证边界见 MMH3-1c。 - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention 和 eager execution；resident layers 只改变 HBM/transfer，不减少 pinned host master。 实现合同与 standalone-audio staging 缺口见 MMH3-3a，5090/4090 证据边界见 MMH3-3b。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 86ae467dc..2fc07a6fc 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5737", "PR #5752", "PR #5764", "PR #5829", vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] +sources: ["PR #5703", "PR #5737", "PR #5752", "PR #5764", "PR #5801", "PR #5829", vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] confidence: high --- @@ -19,6 +19,7 @@ confidence: high | online FP8、`ignored_layers`、component prefix | `MMH3-1a` | `pipeline_minimax_h3.py::_resolve_component_quant_config` → `MiniMaxH3DiTModel` linear prefix | | grouped QKV、fused MLP、weight loader、TP | `MMH3-1a` | `minimax_h3_transformer.py::MiniMaxH3DiTModel.load_weights` → active vLLM loader | | FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | +| RMSNorm、RoPE、96/128 rotary dim、fused backend | `MMH3-1c` | `MiniMaxH3Attention` → shared `RMSNorm`/`RotaryEmbedding` → platform dispatch | | FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | @@ -59,6 +60,22 @@ confidence: high 必填输入，不是量化变量；输入合同演进后应修正 fixture，不能靠跳过 case 或放宽质量 gate 恢复 CI。^[PR #5737] ^[PR #5829] +## MMH3-1c — H3 fused norm/RoPE 必须保持 checkpoint dtype 与 partial-rotation 布局 + +- 触发：修改 H3 q/k norm、RoPE frequency packing、head dim 或 shared fused-layer dispatch。 +- 强制：H3 使用共享 `RMSNorm`，gamma 保持 BF16，而 native reduction/scale 在 FP32 累加后 + cast 回输入 dtype。CUDA/HIP eager 与 MUSA 继承的 CUDA 路径尝试 fused RMSNorm、异常时回退 + native，compile tracing 直接 native；XPU 走 native，NPU 直接调用 `npu_rms_norm` 且没有该 + fallback。RoPE 使用 NeoX `RotaryEmbedding(half_head_dim=False)`：128 维 head 只旋转前 96 维 + 并原样保留末 32 维；CUDA/HIP/native 把 H3 的 tiled full frequencies 转为 half kernel layout， + XPU/MUSA 明确走 native，MindIE 接收 full layout。 +- 禁止：把任意 96 维随机 frequency 当成 H3 producer 合同；合法 full layout 是同一 48 维 + half 的拼接。也不能从 CPU/reference 与 mocked NPU/MindIE wiring 推断真实 fused-kernel parity。 +- 验收：以 `cat([half, half])` frequency 对照 native reference，逐值断言 96 维旋转和 32 维 + passthrough；另测 BF16 gamma/FP32 accumulation、compile native path 与各平台参数 wiring。 + 真实 kernel 性能与数值结论需目标硬件复测；PR 报告的 491014→475070 ms 来自非目标 + `5215e03a` 且缺硬件与重复次数，只能作为有界外部观察，不能写成稳定 3.35% 保证。^[PR #5801] + ## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 - 触发：H3 `t2va`/`fl2va`/`ref2va`、reference 计数、frame index、canvas、duration 或 From cb3357cb32e421af7048c26e4355ab0a9ef82d30 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 15:54:52 +0800 Subject: [PATCH 035/465] knowledge(vllm-omni): sync f879ad49 (PR #5837) Upstream-Commit: f879ad4983855ca89e6c97a2ab0374d319f1cfcc Upstream-PR: #5837 Knowledge-Delta: MiniMax H3 accelerator-aware VAE RNG isolation --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../vllm-omni/models/minimax-h3/_index.md | 9 ++++---- .../vllm-omni/models/minimax-h3/rules.md | 22 ++++++++++++------- 11 files changed, 30 insertions(+), 23 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c23683fc2..f63672329 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 240e6655be1ded9a5635cf98638f325188ce3267 + previous_audited_sha: f2e3d2e25bebc0c7110ccb945d7dcf4be81bf6db audited_ref: main - audited_sha: f2e3d2e25bebc0c7110ccb945d7dcf4be81bf6db + audited_sha: f879ad4983855ca89e6c97a2ab0374d319f1cfcc # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ad51b2279..264afc022 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - f2e3d2e25bebc0c7110ccb945d7dcf4be81bf6db`. The canonical machine baseline + f879ad4983855ca89e6c97a2ab0374d319f1cfcc`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 508998ab4..973069a94 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f2e3d2e2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f879ad49` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b738f4c99..e4f21c2bf 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f2e3d2e2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f879ad49` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e5ffd6f3b..c83e648de 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f2e3d2e2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f879ad49` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0719070ec..39bb1cd74 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f2e3d2e2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f879ad49` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 84eb74ebd..05f295335 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f2e3d2e2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f879ad49` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1049eabb5..197622b9a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f2e3d2e2` 验证存在 +- 源码校验：以上路径均已在 `main @ f879ad49` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d391d86c3..ece435c32 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f2e3d2e2`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f879ad49`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to f2e3d2e2 \ + --to f879ad49 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 4e1330993..58f376afb 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", "PR #5801", "PR #5829", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", "PR #5801", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -41,9 +41,10 @@ confidence: high - 音频加载优先使用 torchaudio；当 TorchCodec/torchaudio 在 CPU-only aarch64 环境不可用 时，`reference_video.load_audio_file` 回退到 soundfile，再对 libsndfile 不支持的格式 通过 ffmpeg 转 WAV。该回退保持 `(channels, samples)` float32 与原始 sample rate 合同。 -- conditioned image/video VAE 用固定内部 seed，并在 `fork_rng` 中播种 CPU 与参数所在设备后 - 恢复 state；MUSA recipe 记录 MTT S5000 验证。目标实现实际接纳所有非 CPU device，而实机 - RNG 证据只覆盖 CUDA/MUSA；支持边界与并发缺口见 MMH3-2c。 +- conditioned image/video VAE 用固定内部 seed，并把 parameter 的真实 `device_type` 传给 + `fork_rng`，以保存/恢复 CPU 与对应 accelerator RNG；MUSA recipe 记录 MTT S5000，PR #5837 + 另报告 Ascend NPU smoke/FL2VA 成功。目标实现接纳已注册的非 CPU device module，但持续 + 回归、XPU/ROCm 与并发边界仍见 MMH3-2c。 - H3 q/k 使用共享 RMSNorm（BF16 gamma、native FP32 accumulation）与 NeoX RoPE；每个 128 维 head 只旋转前 96 维并保留后 32 维，平台布局和验证边界见 MMH3-1c。 - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 2fc07a6fc..420cf140f 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5737", "PR #5752", "PR #5764", "PR #5801", "PR #5829", vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] +sources: ["PR #5703", "PR #5737", "PR #5752", "PR #5764", "PR #5801", "PR #5829", "PR #5837", vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] confidence: high --- @@ -131,19 +131,25 @@ confidence: high - 触发：修改 H3 image/video reference 的 VAE encode、固定 keyframe seed、`fork_rng`、设备 generator 或 accelerator backend 支持。 - 强制：`encode_image()`/`encode_video()` 在采样前暂时把 VAE 转成 FP32，在 - `torch.random.fork_rng(devices=...)` 内同时播种 CPU default generator 与 active-device - generator，退出时恢复 RNG state，并在 `finally` 恢复原 dtype；image path 还必须恢复 - `parallel_tiling`。这个 seed 是 conditioned VAE 的固定内部 seed，不能描述成 request seed。 + `torch.random.fork_rng` 同时传 `devices=...` 与 `device_type=parameter.device.type`，并在 + context 内播种 CPU default generator 与 active-device generator；退出时恢复 RNG state， + 在 `finally` 恢复原 dtype，image path 还必须恢复 `parallel_tiling`。这个 seed 是 + conditioned VAE 的固定内部 seed，不能描述成 request seed。 +- 强制：`devices` 只决定保存/恢复哪些 device index，不能代替 `device_type` 选择 RNG module； + 目标 torch 版本省略后者会默认进入 CUDA。NPU 依赖 `torch_npu` 先注册 `torch.npu`，encode + 内的 `self.device_module` 则在同一 active device 上执行 `device(...)` 与 `manual_seed(...)`。 - 禁止：把 PR 文本中的 CUDA+MUSA allowlist 当成目标实现。目标代码实际以 - `parameter.device.type != "cpu"` 为条件，把所有非 CPU device 交给 `torch.get_device_module()`； - 现有实机证据只覆盖 CUDA/MUSA，不能外推 XPU、NPU 或 ROCm RNG 语义。也不能因 + `parameter.device.type != "cpu"` 接纳已注册的 accelerator device module，并把真实 type + 交给 `fork_rng`；CPU 的 + `devices=[]` 仍保存/恢复 CPU RNG。实机证据覆盖 CUDA/MUSA，以及 PR #5837 报告的 Ascend + NPU direct fork smoke 与一次双参考图 FL2VA serving 成功，不能外推 XPU 或 ROCm。也不能因 `fork_rng` 最终恢复 state 就宣称并发安全：context 内仍会暂时改写 process-global CPU/device generator，重叠调用需要序列化或独立并发证明。 - 验收：同 seed 的 image/video condition latent 可重复，正常和 encode 异常后 CPU、目标设备 RNG state、dtype 及 image tiling state 都恢复；CPU 与每个声称支持的 accelerator 分支分别 覆盖，并加入重叠调用 fence。PR 中拟议的专用 VAE 单测按 review 被删除，目标 commit 没有 - 新增测试；PR body 的 focused/实机结果只能作为外部验证，不能冒充已提交回归覆盖。 - ^[PR #5703] + 新增测试；PR #5837 也只改两处参数，未提交 CPU/NPU 回归测试，其 NPU smoke、serving 结果 + 与 CUDA 不变性说明只能作为外部证据，不能冒充持续回归覆盖。^[PR #5703] ^[PR #5837] ## MMH3-3a — H3 DLO 必须保持 loader layout 与 component stage 配对 From 53f81dcec8ab618bb28ce9d3a013d6603275d010 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 16:01:18 +0800 Subject: [PATCH 036/465] knowledge(vllm-omni): sync bc95e741 (PR #5833) Upstream-Commit: bc95e7415fdea6d3eaf69ee019f37c49890ccf1c Upstream-PR: #5833 Knowledge-Delta: official design navigation and source-map coverage --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/docs/design-doc-map.md | 11 ++++++----- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 10 files changed, 17 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f63672329..8daadbd29 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f2e3d2e25bebc0c7110ccb945d7dcf4be81bf6db + previous_audited_sha: f879ad4983855ca89e6c97a2ab0374d319f1cfcc audited_ref: main - audited_sha: f879ad4983855ca89e6c97a2ab0374d319f1cfcc + audited_sha: bc95e7415fdea6d3eaf69ee019f37c49890ccf1c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 264afc022..1bd298911 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - f879ad4983855ca89e6c97a2ab0374d319f1cfcc`. The canonical machine baseline + bc95e7415fdea6d3eaf69ee019f37c49890ccf1c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 973069a94..fc9e9bf00 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f879ad49` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bc95e741` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e4f21c2bf..3bd865c7d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f879ad49` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bc95e741` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c83e648de..86411d4b2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f879ad49` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bc95e741` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 39bb1cd74..08dcbfbca 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f879ad49` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bc95e741` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 05f295335..f4d836d4e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f879ad49` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bc95e741` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 197622b9a..96231599f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f879ad49` 验证存在 +- 源码校验：以上路径均已在 `main @ bc95e741` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index 6ed114fd8..5d6954745 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -4,15 +4,16 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, docs] -sources: [docs/design/index.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/omni_async_output_materialization.md, docs/user_guide/diffusion/execution_modes.md] +sources: ["PR #5833", docs/.nav.yml, docs/design/index.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/omni_async_output_materialization.md, docs/user_guide/diffusion/execution_modes.md] --- # 官方设计文档地图（docs/design/**） -`main @ 9f978923` 复核。注意：官方 `docs/design/index.md` 只列了子集（Architecture -Overview、4 篇 feature、metrics、3 篇 module）；**完整树比索引大得多**—— -`feature/` 下还有 async_chunk、cache_dit、teacache、prefix_caching、7 篇并行策略、 -`omni_connectors/` 逐后端 spec 等未入索引的文档，找 spec 时直接 `ls docs/design/feature/`。 +`main @ bc95e741` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 已按 architecture、 +runtime/stage、communication、diffusion acceleration、infrastructure/performance 和 module +重组，并纳入 connector、缓存、并行、量化及 async diffusion 等现有入口；此次只改导航与 +landing-page 编排，文档路径保持不变，不代表 runtime 或设计合同发生变化。索引仍未列 +`docs/design/ar_diffusion_pipeline_capability.md`；找 spec 时仍须检查 `docs/design/` 实际文件树。 | 官方文档 | 一句话 | 知识树 owner | |---|---|---| diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ece435c32..7e9d3cf80 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f879ad49`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bc95e741`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to f879ad49 \ + --to bc95e741 \ --repo \ --mode report-only ``` From 6ec6b27c1f736bd854118e9ab94c6c532b7e1cc9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 16:13:47 +0800 Subject: [PATCH 037/465] knowledge(vllm-omni): sync 93457f29 (PR #5706) Upstream-Commit: 93457f29ff88b34b9e1851f207f989c3b1bb6e0b Upstream-PR: #5706 Knowledge-Delta: MiniMax H3 RainFusion attention and NPU INT8 --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/minimax-h3/_index.md | 5 +++- .../vllm-omni/models/minimax-h3/rules.md | 29 ++++++++++++++++++- 11 files changed, 43 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8daadbd29..869fecc7a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f879ad4983855ca89e6c97a2ab0374d319f1cfcc + previous_audited_sha: bc95e7415fdea6d3eaf69ee019f37c49890ccf1c audited_ref: main - audited_sha: bc95e7415fdea6d3eaf69ee019f37c49890ccf1c + audited_sha: 93457f29ff88b34b9e1851f207f989c3b1bb6e0b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1bd298911..596cbef4c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - bc95e7415fdea6d3eaf69ee019f37c49890ccf1c`. The canonical machine baseline + 93457f29ff88b34b9e1851f207f989c3b1bb6e0b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fc9e9bf00..bc44e5144 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bc95e741` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 93457f29` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3bd865c7d..5408d4f6b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bc95e741` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 93457f29` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 86411d4b2..eb9511999 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bc95e741` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 93457f29` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 08dcbfbca..ff2673c21 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bc95e741` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 93457f29` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f4d836d4e..3ca2937c2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bc95e741` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 93457f29` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 96231599f..226f2aeb8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bc95e741` 验证存在 +- 源码校验：以上路径均已在 `main @ 93457f29` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 7e9d3cf80..a25f512f1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bc95e741`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 93457f29`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to bc95e741 \ + --to 93457f29 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 58f376afb..28d8aaa11 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", "PR #5801", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", "PR #5801", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -47,6 +47,9 @@ confidence: high 回归、XPU/ROCm 与并发边界仍见 MMH3-2c。 - H3 q/k 使用共享 RMSNorm（BF16 gamma、native FP32 accumulation）与 NeoX RoPE；每个 128 维 head 只旋转前 96 维并保留后 32 维，平台布局和验证边界见 MMH3-1c。 +- Ascend NPU 可选择 RainFusion 稀疏 video tail，并从 BF16 checkpoint 做 online INT8；两者只在 + exact T2VA/Ulysses/no-AllGather DLO 配置有完成证据，几何 fallback、TP width 与组合边界见 + MMH3-1a/1d。 - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention 和 eager execution；resident layers 只改变 HBM/transfer，不减少 pinned host master。 实现合同与 standalone-audio staging 缺口见 MMH3-3a，5090/4090 证据边界见 MMH3-3b。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 420cf140f..02d17ad1b 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5737", "PR #5752", "PR #5764", "PR #5801", "PR #5829", "PR #5837", vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md] +sources: ["PR #5703", "PR #5706", "PR #5737", "PR #5752", "PR #5764", "PR #5801", "PR #5829", "PR #5837", vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/quantization/int8_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] confidence: high --- @@ -20,6 +20,7 @@ confidence: high | grouped QKV、fused MLP、weight loader、TP | `MMH3-1a` | `minimax_h3_transformer.py::MiniMaxH3DiTModel.load_weights` → active vLLM loader | | FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | | RMSNorm、RoPE、96/128 rotary dim、fused backend | `MMH3-1c` | `MiniMaxH3Attention` → shared `RMSNorm`/`RotaryEmbedding` → platform dispatch | +| RainFusion、block sparse、video layout、NPU INT8 | `MMH3-1d` | packed sequence → attention metadata/backend plan；quant config → prefixed linear → loader/post-load | | FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | @@ -43,6 +44,12 @@ confidence: high 与 gate/up sentinel 断言传给 active loader 的顺序和 shard id，至少覆盖 TP 与 online FP8。 ^[PR #5737] +NPU INT8 online 同样复用 component-relative prefix 与 active loader：BF16 checkpoint 在逐层 +load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection 保持未量化； +`ignored_layers` 必须能用 fused owner 名（如 `blocks.0.attn.qkv_proj`）精确跳过。NPU +`QuantBatchMatmulV3` 的 65535 output 限制只按 per-partition width 判断，超限 layer 回退 +`UnquantizedLinearMethod`，更高 TP 可能让同一层重新满足 INT8。^[PR #5706] + ## MMH3-1b — joint video/audio quality 与 offload 兼容性必须一起验收 - 触发：改变 H3 FP8 layer scope、kernel/loader、quality threshold、offload 或 nightly lane。 @@ -76,6 +83,26 @@ confidence: high 真实 kernel 性能与数值结论需目标硬件复测；PR 报告的 491014→475070 ms 来自非目标 `5215e03a` 且缺硬件与重复次数，只能作为有界外部观察，不能写成稳定 3.35% 保证。^[PR #5801] +## MMH3-1d — RainFusion 只在完整、对齐的 H3 video tail 上稀疏 + +- 触发：修改 `RAINFUSION_ATTN`、`block_sparse`、packed video geometry、attention layout、 + denoise step/layer skip 或 NPU backend resolution。 +- 强制：显式选择时所有平台在构造模型前检查 backend platform/dependency；RainFusion 仅支持 + Ascend NPU、要求 MindIE-SD，且不兼容 Ring（用 Ulysses）。H3 attention 显式声明 BSND， + `VideoTokenLayout(prefix_len, latent_grid)` 必须证明 video 是 packed document 0 的 tail。 +- 强制：只有 sparsity>0、step>=start_step、非 skip layer、video>=32×128 rows 且 video rows + 为 128 的倍数时调用 `rf_v2`；prefix/text/reference/audio 保持 dense。无 layout、无 + `max_seqlen_q`、长度不闭合、短序列、未声明 layout 或未对齐 geometry 一律走 NPU Flash dense + fallback，不能 padding，因为 kernel 忽略 mask，padding key 会改变 softmax denominator。 +- 禁止：把 nominal sparsity 当 realized sparsity或质量保证；未声明 layout 时不能让 sparse path + 假设 BSND 而 dense fallback 解释成 BNSD。INT8、RainFusion、no-AllGather DLO 可组合，但 online + quantization 不得与 DLO+AllGather 组合；本 PR 没有证明 HSDP 或其他 quantizer 的组合语义。 +- 验收：CPU plan tests 覆盖 aligned/misaligned、tail closure、min length、layout 和 skip/step； + NPU 条件数值测试以 `sparsity=0` 直接调用 kernel 对 dense reference，mean relative error 阈值 + 为 `2e-3`；它不证明 `sparsity=0.8` 质量 parity。PR 的 Atlas 800I A3 8×NPU、 + CANN 9.0.1、T2VA 209-frame 三种生成仅证明这些 exact 配置能完成；视频样例不是质量阈值， + 没有 latency/repeats，不能宣称稳定加速或 FL2VA/Ref2VA 支持。^[PR #5706] + ## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 - 触发：H3 `t2va`/`fl2va`/`ref2va`、reference 计数、frame index、canvas、duration 或 From 41fceb13c4997d6a081a18ac453ea433037dcfb5 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 16:20:42 +0800 Subject: [PATCH 038/465] knowledge(vllm-omni): sync a9ba47d8 (PR #5732) Upstream-Commit: a9ba47d8ce557caae567291b3386f287c0f26f81 Upstream-PR: #5732 Knowledge-Delta: bounded video frame conversion memory --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules.md | 23 ++++++++++++++++--- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 10 files changed, 31 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 869fecc7a..5f3369e97 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bc95e7415fdea6d3eaf69ee019f37c49890ccf1c + previous_audited_sha: 93457f29ff88b34b9e1851f207f989c3b1bb6e0b audited_ref: main - audited_sha: 93457f29ff88b34b9e1851f207f989c3b1bb6e0b + audited_sha: a9ba47d8ce557caae567291b3386f287c0f26f81 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 596cbef4c..e45174055 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 93457f29ff88b34b9e1851f207f989c3b1bb6e0b`. The canonical machine baseline + a9ba47d8ce557caae567291b3386f287c0f26f81`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bc44e5144..303e26563 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 93457f29` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a9ba47d8` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5408d4f6b..eef425225 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 93457f29` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ a9ba47d8` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index eb9511999..8c265403a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 93457f29` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a9ba47d8` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ff2673c21..c68ddf3b6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 93457f29` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a9ba47d8` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3ca2937c2..5d3f4ee46 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 93457f29` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a9ba47d8` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 226f2aeb8..486878b6c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 93457f29` 验证存在 +- 源码校验：以上路径均已在 `main @ a9ba47d8` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 71b8ae69f..f5d4a4c5f 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5713", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5713", "PR #5732", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -25,7 +25,7 @@ confidence: high | sleep/wake、partial stage/tag、idempotency、ACK、generation admission | `engine-lifecycle`：`SERV-5a`, `SERV-5b` | `entrypoints/async_omni.py::{AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni.generate}` → `worker/base.py::{handle_sleep_task,handle_wake_task}` / `diffusion/worker/diffusion_worker.py` | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | -| video reference 解码、mixed media、capability、bounded upload、空帧 | `media-ingress`：`SERV-1c/1d` | `entrypoints/openai/video_api_utils.py::{OmniVideoBackend,_decode_video_bytes}`、`serving_video.py::supports_mixed_reference_inputs` → video server callers | +| video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | | Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle | `metrics-lifecycle`：`SERV-2a`, `SERV-2b` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | @@ -33,7 +33,7 @@ confidence: high |---|---|---| | `core` | 每次 serving 审查 | `SERV-4c` | | `streaming-format` | SSE、audio streaming、format/default/capability | `SERV-1a`, `SERV-1b` | -| `media-ingress` | video reference、decoder registry/backend、mixed capability、bounded upload | `SERV-1c`, `SERV-1d` | +| `media-ingress` | video reference、decoder registry/backend、mixed capability、bounded upload/conversion | `SERV-1c`–`1e` | | `metrics-lifecycle` | metrics、gauge、replica、collector | `SERV-2a`, `SERV-2b` | | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c` + 命中模型规则 | @@ -89,6 +89,23 @@ confidence: high 是否同样 bounded 必须单独证明；本 pin 的 helper 仍完整读取。模型专有 allowlist、temp-file 与 input matrix 见 [MiniMax H3 rules](../../models/minimax-h3/rules.md)。 ^[PR #5752] +## SERV-1e — 长视频转换只保留必要的全视频 buffer，并逐帧保持旧语义 + +- 触发：修改 video output 的 normalize、float→uint8、RGBA strip、mux 或 fragmented encode。 +- 强制：预分配最终 contiguous uint8 video，逐帧 clip/scale/round 写入，禁止先 `np.stack` + 全量 float frames 再产生全量转换临时量；这只消除额外副本，原 normalized float frames、最终 + uint8 buffer 及上游 device→host 分配仍存在，不能称为 O(1) memory。 +- 强制：所有 frame shape 相同；只有 rank-3 HWC 且 C=4 才去 alpha，二维 width=4 灰度不能 + 截断。mixed float dtype 先用 `np.result_type` 得到 common dtype，再逐帧计算，以保持 legacy + stack 的 promotion/rounding/checksum；不得原地改输入。当前 uint8 fast branch 因 normalization + 先转 float32 而实际上不可达，不能拿它作为性能或语义证明。 +- 验收：回归测试禁止 float path 调用 `np.stack`，并覆盖 input immutability、RGBA、width-4 + grayscale、mixed float16/float32 与 exact uint8 output。性能证据必须分开报告 conversion 和 + 未改动的 MP4 encode：PR #5732 的可复现实验仅绑定 209×1344×768 float32 RGB、24 FPS、 + ultrafast、fresh process、3 次 median/RSS 10 ms；conversion 1201.15→529.47 ms、conversion+MP4 + 1808.96→1235.26 ms、peak RSS 8.874→4.141 GiB、above-resident 5.393→0.660 GiB，两个 hashes + 相同。PR body 的 362-frame 与 4×MI300X E2E 是另一组观察，不能混合或泛化。^[PR #5732] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a25f512f1..2af849154 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 93457f29`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a9ba47d8`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 93457f29 \ + --to a9ba47d8 \ --repo \ --mode report-only ``` From 4d4425e490ea13954c5a85bc34bc2a337dc9d5d5 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 16:28:01 +0800 Subject: [PATCH 039/465] knowledge(vllm-omni): sync d219d93b (PR #5779) Upstream-Commit: d219d93bbb4db1a93b6c84e77f375838ebd8246e Upstream-PR: #5779 Knowledge-Delta: MiniMax H3 TRTLLM packed attention --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/minimax-h3/_index.md | 4 ++- .../vllm-omni/models/minimax-h3/rules.md | 31 ++++++++++++++++++- 11 files changed, 44 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5f3369e97..e9cc37899 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 93457f29ff88b34b9e1851f207f989c3b1bb6e0b + previous_audited_sha: a9ba47d8ce557caae567291b3386f287c0f26f81 audited_ref: main - audited_sha: a9ba47d8ce557caae567291b3386f287c0f26f81 + audited_sha: d219d93bbb4db1a93b6c84e77f375838ebd8246e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e45174055..fd0aa95df 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - a9ba47d8ce557caae567291b3386f287c0f26f81`. The canonical machine baseline + d219d93bbb4db1a93b6c84e77f375838ebd8246e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 303e26563..3313eaf20 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a9ba47d8` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d219d93b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index eef425225..2d4ae7ead 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a9ba47d8` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d219d93b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8c265403a..3363b979d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a9ba47d8` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d219d93b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c68ddf3b6..e59cad6bb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a9ba47d8` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d219d93b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 5d3f4ee46..3afe79c84 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a9ba47d8` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d219d93b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 486878b6c..6a0e95697 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a9ba47d8` 验证存在 +- 源码校验：以上路径均已在 `main @ d219d93b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2af849154..215d157c0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a9ba47d8`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d219d93b`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to a9ba47d8 \ + --to d219d93b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 28d8aaa11..b7c6c1491 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5785", "PR #5801", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -50,6 +50,8 @@ confidence: high - Ascend NPU 可选择 RainFusion 稀疏 video tail，并从 BF16 checkpoint 做 online INT8；两者只在 exact T2VA/Ulysses/no-AllGather DLO 配置有完成证据，几何 fallback、TP width 与组合边界见 MMH3-1a/1d。 +- datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；packed suffix trim、SAGE 短序列 role、 + mask/metadata fail-closed、AllGather-KV 禁用及性能证据边界见 MMH3-1e。 - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention 和 eager execution；resident layers 只改变 HBM/transfer，不减少 pinned host master。 实现合同与 standalone-audio staging 缺口见 MMH3-3a，5090/4090 证据边界见 MMH3-3b。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 02d17ad1b..bb604ce36 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5737", "PR #5752", "PR #5764", "PR #5801", "PR #5829", "PR #5837", vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/quantization/int8_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] +sources: ["PR #5703", "PR #5706", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5829", "PR #5837", vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/quantization/int8_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] confidence: high --- @@ -21,6 +21,7 @@ confidence: high | FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | | RMSNorm、RoPE、96/128 rotary dim、fused backend | `MMH3-1c` | `MiniMaxH3Attention` → shared `RMSNorm`/`RotaryEmbedding` → platform dispatch | | RainFusion、block sparse、video layout、NPU INT8 | `MMH3-1d` | packed sequence → attention metadata/backend plan；quant config → prefixed linear → loader/post-load | +| TRTLLM、ragged packed metadata、SAGE、Skip-Softmax、Blackwell default | `MMH3-1e` | H3 metadata/roles → TRTLLM packed trim/quant gate → platform default | | FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | @@ -103,6 +104,34 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection CANN 9.0.1、T2VA 209-frame 三种生成仅证明这些 exact 配置能完成；视频样例不是质量阈值， 没有 latency/repeats，不能宣称稳定加速或 FL2VA/Ref2VA 支持。^[PR #5706] +## MMH3-1e — H3 TRTLLM 必须从 packed 结构裁掉 padding 并隔离短序列 role + +- 触发：修改 H3 的 TRTLLM default、packed `cu_seqlens`/mask、SAGE quant、Skip-Softmax、 + token-refiner role 或 denoise timestep context。 +- 强制：H3 只在支持的 datacenter Blackwell SM100/103、head_dim=128、FlashInfer kernel 可用且 + 声明 compatible packed/mask-free path 时默认 dense BF16 TRTLLM；SM120/121、缺依赖、错误 + head dim 或需要任意 mask 的路径保留平台 fallback，默认不自动开启 SAGE/Skip-Softmax。 +- 强制：四个 packed metadata key 必须完整、Q/K batch/terminal 覆盖一致；只接受 prefix-valid + structural padding mask，按 boundary 裁掉无效 suffix 后再 quantize/attention，并把输出补零到 + Ulysses 物理 shape。`[0,N,N]` 必须折叠尾部空 sequence；任意 mask、缺失或矛盾 metadata + fail closed。`attention_mask_free=True` 是 compatible packed-path capability，不表示物理 mask + 永远为空。AllGather-KV 的 local-Q/global-KV metadata 不对称合同在初始化时明确拒绝，真正 + 支持按 review 明确延期。 +- 强制：main DiT 与 `minimax_h3.token_refiner` 使用独立 role；任一 KV sequence 短于 SAGE + `k_block_size` 时该 input 告警并走 dense TRTLLM，不能让 14-token refiner 产生 non-finite， + 也不能因此关闭 main DiT 的 SAGE。H3 将 scheduler 的降序 sigma（1→0）发布为 + `normalized_timestep`；Skip-Softmax 在 sigma 大于 `disabled_until_timestep` 时保持 dense， + 到达或低于阈值才启用。 +- 禁止：把 structural prefix mask 泛化为任意 mask；让 padding 进入 SAGE block quantization； + 把 combined packed-H3 优化视频归因于本 PR 单一改动。请求级 `set_forward_context` wrapper + 会在 `finally` 恢复 prior context，因此 loop 内的 step mutation 不会泄漏到后续请求。 +- 验收：测试覆盖 ragged batch、suffix trim/zero restore、empty terminal、invalid mask/metadata、 + short-role dense fallback、AllGather rejection 与 non-finite regression。PR 的 4×B300 SM103、 + 1248×768、209 frames、50 steps FA4/TRTLLM 对比仅绑定该 prompt/seed/topology：PSNR 27.10、 + SSIM 0.8880 不是质量 gate；83.854→71.990 s diffusion、88.558→76.176 s wall 的 warmed A/B + 来自 review follow-up 外部分支 commit `20cc23ae`，其 artifacts 明确是 combined packed-H3 + optimization evidence，并非目标 commit 的隔离实验，不能泛化为稳定 14% 保证。^[PR #5779] + ## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 - 触发：H3 `t2va`/`fl2va`/`ref2va`、reference 计数、frame index、canvas、duration 或 From 987b60c5d6a1b2c4f56fb9442f0a9efe32a2c1de Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 16:42:07 +0800 Subject: [PATCH 040/465] knowledge(vllm-omni): sync 48298030 (PR #5720) Upstream-Commit: 48298030d4b3c320b858dd98934c01babdd320ce Upstream-PR: #5720 Knowledge-Delta: MiniMax H3 modular multi-DiT pipeline --- adapters/vllm_omni/release_baseline.yaml | 8 ++--- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 16 ++++++++-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/minimax-h3/_index.md | 17 +++++++---- .../vllm-omni/models/minimax-h3/rules.md | 30 ++++++++++++++++++- 12 files changed, 67 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e9cc37899..17dccb1a5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a9ba47d8ce557caae567291b3386f287c0f26f81 + previous_audited_sha: d219d93bbb4db1a93b6c84e77f375838ebd8246e audited_ref: main - audited_sha: d219d93bbb4db1a93b6c84e77f375838ebd8246e + audited_sha: 48298030d4b3c320b858dd98934c01babdd320ce # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 79 sha256: 700ab6a4918de2d796604d1e6da5f619731a53bb09b8c56416f253e820ffdf3a diffusion: - count: 58 - sha256: 7dc6a5a373ca33a31a44961c1422008327cede3b364577fa095fd37c05ea2085 + count: 59 + sha256: b6ee711d67f4c5daca6b1c6b5d7091bd61e1f978cb6a6548b79ff5700cafa041 pipelines: count: 52 sha256: 54a31adef93bc33e2009252699075930ee05ffb38605a52ba9370a21c542b320 diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fd0aa95df..dd6d7049f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - d219d93bbb4db1a93b6c84e77f375838ebd8246e`. The canonical machine baseline + 48298030d4b3c320b858dd98934c01babdd320ce`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3313eaf20..a3ea79f54 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d219d93b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 48298030` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2d4ae7ead..7ef769061 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d219d93b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 48298030` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index efebb4694..ddffde682 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/offloader/, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] confidence: high --- @@ -27,6 +27,7 @@ confidence: high | HSDP/FSDP、`fully_shard`、DeviceMesh、packed/scalar parameter、FP8 | `checkpoint-distributed`：`DIFF-2b` | `distributed/hsdp.py::{apply_hsdp_to_model,shard_model}` → loader `_load_model_with_hsdp` → `hsdp_fp8.py` | | distributed layerwise offload、AllGather、异构 block、shared buffer | `checkpoint-distributed`：`DIFF-2e` | `offloader/distributed_layerwise_backend.py::{DistributedLayerwiseOffloadHook.initialize_hook,prefetch_layer,DistributedLayerwiseOffloadBackend._allocate_shared_buffers}` | | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `data.py::_propagate_quantization_from_tf_config` → loader weight sources/post-load → component linear consumer | +| modular/multi-DiT、`_dit_modules`、Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f` | pipeline runtime component list → registry/runner/cache/LoRA/module discovery consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | paged KV/cache、backend/platform、GQA/layout、预算与 admission | `system-runtime`：`DIFF-4a`–`4f` | engine init → `diffusion_kv/{initialization,paged_attention_adapter}.py` → platform hook → scheduler | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | @@ -35,7 +36,7 @@ confidence: high |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、async output readiness | `DIFF-1a`–`1e` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload | `DIFF-2a`, `DIFF-2b`, `DIFF-2c`, `DIFF-2d`, `DIFF-2e` | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT lifecycle | `DIFF-2a`–`2f` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4f` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | @@ -176,6 +177,17 @@ confidence: high DP concurrent wave 只可在 DLO+AllGather 开启，并从 `sampling_params.extra_args` 比较 canonical signature；no-AllGather 的 TP-local request 不得误走 fused batch。^[PR #5764] +### DIFF-2f — 多 DiT pipeline 的 component list 是所有共享 lifecycle 的唯一集合 + +- 触发：pipeline 增加第二个 DiT、task-dependent transformer，或修改 `_dit_modules`。 +- 强制：registry SP injection、loader strictness、Cache-DiT enable/refresh/summary、regional + compile、LoRA、offloader/module collection 都遍历运行时实际存在的 `_dit_modules`；允许实例按 + task 收窄 list，不得由 class 默认预建不存在的组件。模型规则只补 task/权重选择与 residency。 +- 禁止：硬编码 `transformer`/`transformer_2` 或把非 canonical attribute 静默漏掉；不能用 + 第一个 DiT 成功推断第二个已启用 cache/compile/offload。 +- 验收：用非 canonical 第二属性覆盖每个 consumer 的 enable/refresh/summary/discovery，并覆盖 + task 收窄后的缺省属性；至少一个真实双 DiT pipeline 逐 route 验证。^[PR #5720] + ## 质量阈值与资源辅助 ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3363b979d..f3286c5f9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d219d93b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 48298030` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e59cad6bb..49d775162 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d219d93b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 48298030` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3afe79c84..63b372156 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d219d93b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 48298030` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6a0e95697..15d9a9120 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d219d93b` 验证存在 +- 源码校验：以上路径均已在 `main @ 48298030` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 215d157c0..f750be630 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d219d93b`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 48298030`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to d219d93b \ + --to 48298030 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index b7c6c1491..11d991dff 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -14,8 +14,14 @@ confidence: high - checkpoint：`MiniMaxAI/MiniMax-H3`；纯 diffusion registry architecture 是 `MiniMaxH3Pipeline`，实现位于 `diffusion/models/minimax_h3/`。 -- 支持 `t2va`、`fl2va`、`ref2va` 三种 joint video+audio 条件模式；一个 server 进程 - 一次只加载 `FL2VA` 或 `Ref2VA` 分区，不能把两个分区当成同一份权重同时服务。 +- root `modular_model_index.json` 可解析为 `MiniMaxH3ModularPipeline` alias：combined 默认加载 + FL2VA/Ref2VA 两套 DiT 并共享 text encoder 与 VAE；`--task-type` 可限制到单分区，完整 + discovery、task、capability 与多 DiT lifecycle 合同见 MMH3-2d。目标 pin 的 modular alias + 遗漏 TRTLLM auto-default 所需 capability，默认 combined 服务需显式 backend 或先修 metadata。 +- combined root 同一进程服务 `t2va`、`fl2va`、`ref2va`；启动时 `--task-type t2va/fl2va` + 选择 FL2VA-only，`ref2va` 选择 Ref2VA-only。combined 省略 request task 时，无媒体→t2va、 + image-only→fl2va、video/audio→ref2va（因此 image-only Ref2VA 必须显式 task）；Ref2VA-only + 省略时固定→ref2va。 - 输出是带同步音频的 MP4；FL2VA 接受首帧、尾帧或有序首尾帧，Ref2VA 支持 image-only 及有 visual reference 的 image/video/audio mixed matrix。完整计数、媒体和 API 合同见 MMH3-2a/2b。 @@ -72,9 +78,8 @@ T2VA full-model accuracy 入口在 `tests/e2e/accuracy/minimax_h3/`：模型 sna PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、text-encoder TP4 和 VAE patch parallel 4。该用例是精度/媒体合同，不是性能基线。 硬件 recipe 只记录已验证的 GPU/NPU 形状；性能数字不能从 recipe 的配置示例泛化为全硬件 -保证。目标 pin 的 MUSA recipe 把 launch 描述成 validated Ref2VA 配置，却在命令中导出 -`MODEL="${MODEL_ROOT}/FL2VA"`；按分区合同它不能服务 `task=ref2va`，执行示例前必须改为 -`Ref2VA` 分区，不能把原 snippet 当成可运行的 Ref2VA profile。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 +保证。目标 pin 的 MUSA Ref2VA recipe 已改为 `MODEL_ROOT/Ref2VA` + `--task-type ref2va`； +它只验证单分区启动，不能外推 combined 双 DiT 的容量或性能。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 [Configuration](../../components/configuration/_index.md) 和 [Serving](../../components/serving/_index.md)。 ## 审查入口 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index bb604ce36..28852b71f 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5829", "PR #5837", vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/quantization/int8_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5829", "PR #5837", vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/quantization/int8_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] confidence: high --- @@ -25,6 +25,7 @@ confidence: high | FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | +| modular checkpoint、combined/partition task、两套 DiT、shared component | `MMH3-2d` | model index discovery → startup task selection → task-specific transformer/cache lifecycle | | DLO、TP-local、resident layers、encoder/VAE staging | `MMH3-3a` | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | `MMH3-3b` | recipe measurement commit/run record → exact target topology → quality/capacity validation | @@ -207,6 +208,33 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection 新增测试；PR #5837 也只改两处参数，未提交 CPU/NPU 回归测试，其 NPU smoke、serving 结果 与 CUDA 不变性说明只能作为外部证据，不能冒充持续回归覆盖。^[PR #5703] ^[PR #5837] +## MMH3-2d — modular H3 的 task selector 必须同步权重、能力与所有 DiT lifecycle + +- 触发：修改 `modular_model_index.json`、`MiniMaxH3ModularPipeline`、`--task-type`、combined + FL2VA/Ref2VA 服务、`_dit_modules` 或 shared text/VAE components。 +- 强制：模型发现同时识别 `model_index.json` 与 `modular_model_index.json`，registry alias 与 + postprocess 都映射到同一 H3 pipeline。`auto/combined` 从 root 加载 FL2VA `transformer` 与 + Ref2VA `transformers_ref`，但共享 text encoder、video/audio VAE；单 task 只加载所需 DiT。 + request task 必须属于启动时 `supported_tasks`；combined 省略 task 时按无媒体→t2va、image-only + →fl2va、video/audio→ref2va 推断，因此 image-only Ref2VA 必须显式给 task；Ref2VA-only + 省略 task 时保留 implicit ref2va。 +- 强制：modular alias 复制 9-image/mixed-reference admission capability；所有基于 + `_dit_modules` 的 consumer（loader strictness、Cache-DiT enable/refresh/summary、LoRA/offload） + 遍历实际 DiT，不硬编码 `transformer`/`transformer_2`。共享 `--task-type` 放宽后，非 H3 + owner 仍 model-aware 校验；Qwen3-TTS 只接受既有三值。 +- 缺口：目标 pin 的 modular metadata 虽修复 admission 字段，仍遗漏 canonical H3 的 + `attention_mask_free=True`。HF root 的两个 index 都解析成 modular alias，因此默认 + combined 服务在 Blackwell 不会自动选 TRTLLM，与 recipe 声明冲突；alias 必须校验全部能力 + 字段 parity，不能只复制 review 点名的字段。 +- 禁止：从 combined 注册推断双 DiT 性能已验证；recipe 最终只描述配置，没有 combined + warm latency/throughput/output-parity qualification。也不能用单 DiT Cache-DiT 测试证明 + `transformers_ref` lifecycle。 +- 验收：覆盖 local/Hub index discovery、snapshot allow-pattern、combined/单分区初始化、缺 + partition、task default/membership、modular admission、两 DiT cache lifecycle 与非法 TTS + selector。PR body 的顺序 T2VA→Ref2VA 视频使用 `duration=2.0`，低于目标 pin 已生效的 4 秒 + 下限，因此不能证明 final merge 的 combined 路径；148.27/158.67 GiB combined 与 86.27 GiB + single-load 等数字来自缺硬件/协议/重复次数的 issue comment，不能作为容量保证。^[PR #5720] + ## MMH3-3a — H3 DLO 必须保持 loader layout 与 component stage 配对 - 触发：H3 `OffloadPlan`、`dlo_no_use_allgather`、resident layer、text encoder/VAE staging From ae192fd5f413509ed77032a653e4a2186a9c2e2d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 16:51:39 +0800 Subject: [PATCH 041/465] knowledge(vllm-omni): sync 4b625d11 (PR #5845) Upstream-Commit: 4b625d1175e367e313ef9166a5d296d558fcfce4 Upstream-PR: #5845 Knowledge-Delta: hardware-nested per-sweep DFX baselines --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 35 ++++++++++++------- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- 10 files changed, 33 insertions(+), 24 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 17dccb1a5..5fa845032 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d219d93bbb4db1a93b6c84e77f375838ebd8246e + previous_audited_sha: 48298030d4b3c320b858dd98934c01babdd320ce audited_ref: main - audited_sha: 48298030d4b3c320b858dd98934c01babdd320ce + audited_sha: 4b625d1175e367e313ef9166a5d296d558fcfce4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index dd6d7049f..e1c0c7b3b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 48298030d4b3c320b858dd98934c01babdd320ce`. The canonical machine baseline + 4b625d1175e367e313ef9166a5d296d558fcfce4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 84e20105d..b66a82dc3 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] +sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5845", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] confidence: high --- @@ -93,15 +93,24 @@ confidence: high ## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 - 触发：修改 perf JSON 的 `baseline`、硬件 label/marker、benchmark result schema，或恢复性能阈值断言。 -- 强制：当前 runner 用 `copy.deepcopy` 将输入中的原始 `baseline` 对象原样写入 result artifact； - 硬件 marker/resource label 只路由执行，runner 不按它选择 baseline，也不按 sweep/concurrency - 解析阈值。新配置应使用 `{hardware: {metric: threshold}}` 的硬件嵌套结构，并把缺失 bucket - 当作显式缺口；下游报告工具可以另行选择 bucket，但须声明选择合同。 -- 禁止：把 result 中存在 baseline 描述成 active regression gate；从 `npu: "A3"` marker 推断存在 - A3 阈值；声称当前 runner 实现了文档所说的“按 runtime hardware label 选择”或 flat baseline - 兼容。目标 pin 中这些文档措辞是滞后/前瞻描述：选择与断言 helper、`--assert-baseline` 和 - `skip-performance-assertion` 已删除，flat 输入至多会被不透明地复制，未被 runner 解释。 -- 验收：当前 CI 只用 `completed == num_prompt(s)` gate 请求完成数，baseline artifact round-trip - 保持输入结构和值；若重新引入性能 gate，必须有显式 consumer、硬件 bucket 选择、metric - directionality 与失败测试。目标 pin 的已迁移 baseline bucket 全是 H100；九个 A3-marked 配置 - 没有任何 A3 baseline bucket，这是待补 gap，不是 NPU 性能 gate。^[PR #5402] +- 强制：baseline 只接受 `{hardware: {metric: threshold}}` 的硬件嵌套结构；顶层 hardware 必须是 + `pyproject.toml` 中带 `[hardware-resource]` 标签的 pytest marker，每个 bucket 非空。metric 名可 + 自定义，值只能是单点 scalar 或与 `request_rate` / `max_concurrency` 顺序严格对齐的 list。 + runner 在每个 sweep step 写 result 前按 `sweep_index` 将所有 hardware bucket 的 list 收窄为 + 当前 scalar，scalar 原样保留；结果仍保留所有 bucket，留给下游 consumer 选择实际硬件。 +- 禁止：把 result 中存在 baseline 描述成 active regression gate；从执行 marker 推断同名阈值 + bucket 存在；使用 flat metric map、以 concurrency 为键的 metric dict、未知/alias-only hardware + label，或让 baseline list 与 sweep 顺序脱节。`_RUNTIME_DEVICE_ALIASES` 只服务 result filename + 与运行时身份，不扩展 baseline allowlist；runtime 也不自动选择 baseline bucket。 +- 验收：当前 CI 仍只用 `completed == num_prompt(s)` gate 请求完成数；baseline artifact 是描述性 + 元数据而非阈值断言。单测须覆盖 omni/diffusion sweep 的逐步收窄、所有 hardware bucket 保留、 + 自定义 metric/scalar，以及 flat、未知 label、dict value、缺 index、越界失败。若重新引入性能 + gate，必须有显式 consumer、hardware bucket 选择、metric directionality 与失败测试。 + 当前实现没有 upfront list-length 校验：短 list 到被访问的 index 才失败，多余值静默不用；若 + QPS 与 concurrency 同时配置，同一 list 会按两个独立 loop 的位置复用。因此配置 review 必须 + 人工核对 mode、list 长度和顺序。`[hardware-resource]` 也只代表 baseline key 可解析，不代表 + `hardware_marks()` 能调度该资源：当前 H200/B200 尚未进入 CUDA resource 分支。另有四个 + Hunyuan 配置只保留 H200 baseline，却仍标记 H100/A3 执行，当前因无阈值断言而被掩盖。 + runtime alias 匹配还应保持最长 token 优先；目标 pin 中 `B200` 早于 `GB200`，会把 GB200 名称 + 归一化成 B200，修复前不可把 filename label 当精确设备证据。 + ^[PR #5402] ^[PR #5845] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a3ea79f54..8e7bcface 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 48298030` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4b625d11` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7ef769061..4a574333d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 48298030` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4b625d11` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f3286c5f9..668a49142 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 48298030` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4b625d11` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 49d775162..cd16889f4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 48298030` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4b625d11` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 63b372156..60b822077 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 48298030` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4b625d11` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 15d9a9120..65d05c930 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 48298030` 验证存在 +- 源码校验：以上路径均已在 `main @ 4b625d11` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f750be630..3e700f80d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 48298030`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4b625d11`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 48298030 \ + --to 4b625d11 \ --repo \ --mode report-only ``` From abf8baff4451a8582c0e40cb154321ac4c87d7ed Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 17:01:54 +0800 Subject: [PATCH 042/465] knowledge(vllm-omni): sync 62bf8d63 (PR #5839) Upstream-Commit: 62bf8d630d98a231924f6219b9f53e2ce2879d5b Upstream-PR: #5839 Knowledge-Delta: DLO compatibility and quantization design --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 24 ++++++++++++++++--- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/docs/design-doc-map.md | 16 ++++++++----- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 11 files changed, 42 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5fa845032..44e4fdfd4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 48298030d4b3c320b858dd98934c01babdd320ce + previous_audited_sha: 4b625d1175e367e313ef9166a5d296d558fcfce4 audited_ref: main - audited_sha: 4b625d1175e367e313ef9166a5d296d558fcfce4 + audited_sha: 62bf8d630d98a231924f6219b9f53e2ce2879d5b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e1c0c7b3b..af34c60a7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 4b625d1175e367e313ef9166a5d296d558fcfce4`. The canonical machine baseline + 62bf8d630d98a231924f6219b9f53e2ce2879d5b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8e7bcface..1feb31646 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4b625d11` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 62bf8d63` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 4a574333d..5ffc4b4d0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4b625d11` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 62bf8d63` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index ddffde682..594b58af4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5839", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] confidence: high --- @@ -26,7 +26,7 @@ confidence: high | host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | | HSDP/FSDP、`fully_shard`、DeviceMesh、packed/scalar parameter、FP8 | `checkpoint-distributed`：`DIFF-2b` | `distributed/hsdp.py::{apply_hsdp_to_model,shard_model}` → loader `_load_model_with_hsdp` → `hsdp_fp8.py` | | distributed layerwise offload、AllGather、异构 block、shared buffer | `checkpoint-distributed`：`DIFF-2e` | `offloader/distributed_layerwise_backend.py::{DistributedLayerwiseOffloadHook.initialize_hook,prefetch_layer,DistributedLayerwiseOffloadBackend._allocate_shared_buffers}` | -| component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `data.py::_propagate_quantization_from_tf_config` → loader weight sources/post-load → component linear consumer | +| component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | | modular/multi-DiT、`_dit_modules`、Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f` | pipeline runtime component list → registry/runner/cache/LoRA/module discovery consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | paged KV/cache、backend/platform、GQA/layout、预算与 admission | `system-runtime`：`DIFF-4a`–`4f` | engine init → `diffusion_kv/{initialization,paged_attention_adapter}.py` → platform hook → scheduler | @@ -112,6 +112,12 @@ confidence: high - 验收：测试覆盖已消费 key、未知 key、当前版本不支持的 scale 以及两条 resolution path 的输出名；fused mixed-FP8 还须覆盖 weight/scale/shard、gate/up 顺序、nested serialized FP8 和 combined projection，缺 shard 不得默认 0。 ^[PR #5087] ^[PR #5737] ^[PR #5848] +- 强制：读取 transformer-local `config.json` 时，disk method 与 active method 归一化后不一致 + 必须失败；disk 声明 serialized、`data_type=mx_fp` 或不同 `ignored_layers` 时应按该 component + 重建 config，不能用 online/default 配置解释 serialized 权重。目标 pin 的 + `resolve_quant_config_from_disk` 只有 Wan2.2 一个 live caller，且无 helper 直测；在补齐多模型 + caller 与 mismatch/serialized/ignored-layers 测试前，这只是局部 loading guard，不是全局自动 + checkpoint reconciliation 支持。^[PR #5839] ### DIFF-2b — HSDP/FSDP 修复必须执行真实 fully_shard @@ -129,6 +135,9 @@ confidence: high 保留 `text_encoder` 等完整 component owner prefix 到持权 layer，要么 pipeline 先 resolve 单一 component，再让内部 layer 使用 component-relative prefix。只量化明确支持的 linear， embedding、LM head、patch/final projection 等排除项必须显式保持未量化。 +- 强制：`ComponentQuantizationConfig.resolve()` 只对 `get_quant_method` 收到的 runtime prefix 做 + longest-prefix match，未命中才落到 default；它不会自动感知 checkpoint `WeightsMapper`。 + 因此 mapper、pipeline 与 layer 必须显式维持同一 namespace，并测试重叠 prefix 的最长匹配。 - 禁止：半途裁掉 owner prefix；resolve component 后又要求 ignored layer 带 owner；用一个 组件配置隐式覆盖其他组件；因为同属一个模型就量化全部 linear。 - 验收：至少覆盖“只量化一个组件、其他组件保持 BF16”的真实构造与加载，逐层断言 @@ -161,6 +170,13 @@ confidence: high 的最大 block 分配，但每个 hook 调 collective 前必须按 dtype 截成自己的 `_ag_output_sizes[dtype] == dp_size * local_shard.numel()` 前缀；block-flat-relative repoint offset 只能在这段实际 block buffer 内解释。 +- 强制：AllGather 权重组复用既有 topology：DP>1 优先 DP group；DP=1 且 SP>1 才用 SP group。 + TP 不作为 DLO AllGather group；进入 mmap path 时 TP>1 必须因绕过 TP-aware weight-loader + callback 而拒绝。no-AllGather 只关闭 DLO 新增的权重 collective，不关闭 standard loader + 已建立的 TP/HSDP/SP 语义，也不提供跨 rank host-weight 节省。DP=SP=1 不能从“无 process + group”推断为可用的 rank-local AllGather 模式：目标 pin 的 loader 在 DLO+AllGather、支持 mmap + 时会跳过 `load_weights()`，但 backend 只有 `dp_size>1` 才执行 mmap load，两个 gate 不一致； + 修复并回归前单 rank 应使用 no-AllGather 或普通 layerwise offload。 - 强制：非 DiT component 的 staging 必须由 `OffloadPlan` 显式声明：encoder block group rank-local streaming 不借用 DiT AllGather group，on-demand component 由 pipeline 在真实 encode/decode stage 成对 load/offload，resident layer 只对声明的 DiT path 生效。no-AllGather @@ -175,7 +191,9 @@ confidence: high 当前回归用 mocked collective 检查 8/32-element、dp=2 合同；没有真实 multi-rank collective 覆盖，PR 的 8×NPU H3 E2E 说明也缺少已填写的 commit 证据，不能升级为验证结论。 ^[PR #5802] DP concurrent wave 只可在 DLO+AllGather 开启，并从 `sampling_params.extra_args` 比较 canonical - signature；no-AllGather 的 TP-local request 不得误走 fused batch。^[PR #5764] + signature；no-AllGather 的 TP-local request 不得误走 fused batch。文档 compatibility matrix 中 + “accepted” 只代表配置/源码 guard：TP+no-AllGather、HSDP+SP+no-AllGather、DP+SP 等尚无完整 + model×hardware E2E 数值覆盖，不得升级为 production support。^[PR #5764] ^[PR #5839] ### DIFF-2f — 多 DiT pipeline 的 component list 是所有共享 lifecycle 的唯一集合 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 668a49142..b110cd0ae 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4b625d11` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 62bf8d63` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cd16889f4..94734f199 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4b625d11` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 62bf8d63` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 60b822077..89ba3fdbd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4b625d11` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 62bf8d63` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 65d05c930..297ef5623 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4b625d11` 验证存在 +- 源码校验：以上路径均已在 `main @ 62bf8d63` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index 5d6954745..7d29dd34c 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -4,15 +4,18 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, docs] -sources: ["PR #5833", docs/.nav.yml, docs/design/index.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/omni_async_output_materialization.md, docs/user_guide/diffusion/execution_modes.md] +sources: ["PR #5833", "PR #5839", docs/.nav.yml, docs/design/index.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] --- # 官方设计文档地图（docs/design/**） -`main @ bc95e741` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 已按 architecture、 -runtime/stage、communication、diffusion acceleration、infrastructure/performance 和 module -重组，并纳入 connector、缓存、并行、量化及 async diffusion 等现有入口；此次只改导航与 -landing-page 编排，文档路径保持不变，不代表 runtime 或设计合同发生变化。索引仍未列 +`main @ 62bf8d63` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 按 architecture、runtime/stage、 +communication、diffusion acceleration、infrastructure/performance 和 module 编排。本轮新增 DLO 与 +quantization design，`cpu_offload_diffusion.md` 重命名为 `cpu_offload.md` 并保留 MkDocs redirect， +同时移除 `ray_based_execution.md`，即删除一项误导性的 supported-design surface。目标 pin 虽仍暴露 +`vllm_omni/distributed/ray_utils/`、`worker_backend` 与 `ray_address`，但 diffusion executor 明确 +`NotImplementedError("ray backend is not yet supported.")`，placement-group/actor helpers 也没有 +live caller；这些残留不是 runtime support 证据。索引仍未列 `docs/design/ar_diffusion_pipeline_capability.md`；找 spec 时仍须检查 `docs/design/` 实际文件树。 | 官方文档 | 一句话 | 知识树 owner | @@ -28,7 +31,8 @@ landing-page 编排，文档路径保持不变，不代表 runtime 或设计合 | `feature/async_chunk.md` | 跨 stage 分块流式 | [distributed](../components/distributed/async-chunk.md) | | `feature/cache_dit.md`、`teacache.md`、`prefix_caching.md` | 缓存加速 | [diffusion](../components/diffusion/cache-acceleration.md) | | `feature/{tensor,pipeline,sequence,expert,cfg,vae}_parallel.md`、`hsdp.md` | 并行策略 | [diffusion](../components/diffusion/parallelism.md) | -| `feature/ray_based_execution.md` | Ray vs MP 后端、多机集群 | [serving](../components/serving/_index.md)（暂无专页） | +| `feature/distributed_layerwise_offload.md` + `user_guide/diffusion/cpu_offload.md` | DLO 的 DP/SP/TP/HSDP compatibility、AllGather 与 rank-local loader 合同；部分组合仅有配置级验证 | [diffusion rule DIFF-2e](../components/diffusion/rules.md) | +| `feature/quantization.md` + `user_guide/quantization/overview.md` | diffusion quantization factory、component routing、checkpoint metadata 与验证边界 | [diffusion rule DIFF-2c](../components/diffusion/rules.md) | | `metrics.md`、`qwen3_omni_tts_performance_optimization.md` | Prometheus 指标；TTS 性能优化实录 | [qwen-omni](../models/qwen-omni/architecture.md)（perf 部分） | | `docs/configuration/*` | 配置 schema spec | [configuration](../components/configuration/architecture.md) | | `docs/contributing/ci/*` | L1–L5 与 markers | [ci guides](../ci/guides/test-tiers.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3e700f80d..916b8bdd6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4b625d11`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 62bf8d63`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 4b625d11 \ + --to 62bf8d63 \ --repo \ --mode report-only ``` From ca33315da5d8099562011622e60367b0895daa26 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 17:12:02 +0800 Subject: [PATCH 043/465] knowledge(vllm-omni): sync a874b8e0 (PR #5836) Upstream-Commit: a874b8e09bb5251bb16e3de8bc22edf3880eec09 Upstream-PR: #5836 Knowledge-Delta: MiniMax H3 four-H100 DFX performance lane --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 7 ++-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/minimax-h3/rules.md | 33 ++++++++++++++++++- 12 files changed, 49 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 44e4fdfd4..5e81375df 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4b625d1175e367e313ef9166a5d296d558fcfce4 + previous_audited_sha: 62bf8d630d98a231924f6219b9f53e2ce2879d5b audited_ref: main - audited_sha: 62bf8d630d98a231924f6219b9f53e2ce2879d5b + audited_sha: a874b8e09bb5251bb16e3de8bc22edf3880eec09 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index af34c60a7..bf8de3b4c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 62bf8d630d98a231924f6219b9f53e2ce2879d5b`. The canonical machine baseline + a874b8e09bb5251bb16e3de8bc22edf3880eec09`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index b66a82dc3..230aec0d7 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5845", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] +sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1feb31646..7f1094af5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 62bf8d63` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a874b8e0` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5ffc4b4d0..1ab8dcd22 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 62bf8d63` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ a874b8e0` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 594b58af4..304433180 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -192,8 +192,11 @@ confidence: high 覆盖，PR 的 8×NPU H3 E2E 说明也缺少已填写的 commit 证据，不能升级为验证结论。 ^[PR #5802] DP concurrent wave 只可在 DLO+AllGather 开启，并从 `sampling_params.extra_args` 比较 canonical signature；no-AllGather 的 TP-local request 不得误走 fused batch。文档 compatibility matrix 中 - “accepted” 只代表配置/源码 guard：TP+no-AllGather、HSDP+SP+no-AllGather、DP+SP 等尚无完整 - model×hardware E2E 数值覆盖，不得升级为 production support。^[PR #5764] ^[PR #5839] + “accepted” 默认只代表配置/源码 guard；PR #5836 的 predecessor partition-path commit 为 H3 + USP4+HSDP4+no-AllGather+8 resident layers 提供一次 4×H100、3/3 completion/perf run，故该 exact + 组合不再是纯 config evidence，但没有 ordinary-offload 数值/输出质量对照，且 profiler 开启、 + final repo-root modular route 未复测，仍不得升级为 production support。TP+no-AllGather、一般 + DP+SP 等其他组合仍缺完整 model×hardware E2E。^[PR #5764] ^[PR #5839] ^[PR #5836] ### DIFF-2f — 多 DiT pipeline 的 component list 是所有共享 lifecycle 的唯一集合 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b110cd0ae..58919d4a7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 62bf8d63` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a874b8e0` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 94734f199..bda3cf318 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 62bf8d63` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a874b8e0` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 89ba3fdbd..b2f1bead8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 62bf8d63` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a874b8e0` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 297ef5623..bcc73f9f7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 62bf8d63` 验证存在 +- 源码校验：以上路径均已在 `main @ a874b8e0` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 916b8bdd6..111e6b63c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 62bf8d63`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a874b8e0`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 62bf8d63 \ + --to a874b8e0 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 28852b71f..f99bdb46d 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5829", "PR #5837", vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/quantization/int8_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5829", "PR #5836", "PR #5837", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] confidence: high --- @@ -28,6 +28,7 @@ confidence: high | modular checkpoint、combined/partition task、两套 DiT、shared component | `MMH3-2d` | model index discovery → startup task selection → task-specific transformer/cache lifecycle | | DLO、TP-local、resident layers、encoder/VAE staging | `MMH3-3a` | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | `MMH3-3b` | recipe measurement commit/run record → exact target topology → quality/capacity validation | +| 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | `MMH3-3c` | nightly lane → perf JSON → benchmark request encoder/result artifact | ## MMH3-1a — component namespace 与 checkpoint transform 必须在 active loader 前闭合 @@ -269,5 +270,35 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection `examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh` 实际不存在，因此不能把其 four-task/MP4 validation 描述成可执行入口；补文件或改文档后再验收。^[PR #5764] +## MMH3-3c — H100 DFX fixture 只证明 exact nightly workload 与 payload path + +- 触发：引用 H3 4×H100 latency、throughput、peak memory、DLO 节省，或修改 X2V perf lane、 + random video reference 与 model-path resolution。 +- 强制：H100 测量只绑定 PR 中较早的 partition-path commit `e8c343d5`：4×H100 80 GiB、 + USP4+HSDP4+encoder TP4+VAE PP4 tile、FLASH_ATTN、1344×768、209 frames、24 fps、8 steps、 + seed 42、1 warmup、3 measured prompts、concurrency 1。后续 `4767c524` 才将 final cases 改成 + repo-root modular model + `--task-type`，PR 没展示 switch 后复测；final nightly 只是预期采集。 + target config 中 T2V/TI2V/V2V/DLO baseline 分别是 latency mean + 38.3252/38.1385/105.7508/38.2089 s，peak-memory mean 54219.6/54622.8/55180.8/36218.4 MB； + 它们是 result metadata，当前 completed-count gate 不消费这些阈值，不能称为 regression guard。 + predecessor partition-path 实测值为 42.5836/42.3761/117.5009/42.4543 s、 + 60244/60692/61312/40242.67 MB、 + 0.0235/0.0236/0.0085/0.0236 qps，且 profiler 开启；不得外推为 no-profiler/production 数字。 + config 对三类 metric 都存 0.9×实测；对越小越好的 latency/memory，这不是“允许 +10%”的 + 正确 upper bound，即使未来恢复断言也必须先修 directionality/threshold。 +- 强制：Ref2VA synthetic clip 先用 OpenCV 生成，再尽力用 ffmpeg/libx264 转 H.264；H3 request + 必须走 `video_reference` data URL，使 server 持久化 `source_path` 给 ffprobe/ffmpeg consumer。 + 缺 ffmpeg 时 fallback `mp4v`，但 H3 要求 H.264/H.265，因此该 fallback 不是有效 H3 兼容路径。 +- 禁止：把上述 predecessor 数字当成 final repo-root modular route 的实测，或泛化到 50-step、 + 并发及其他 topology；final diff 也没有 PR body 所述 RoPE 代码修复。特殊 payload + 当前靠 resolved model 字符串包含大小写敏感的 `MiniMax-H3` 判断；materialized env/cache path + 若不含该 token 会退回 multipart `input_reference`，所以 lane 接入本身不证明 V2V 稳定执行。 +- 验收：nightly 必须证明四个 pytest case 均实际 collected、3/3 completed、artifact/log 上传， + 并在 materialized root、offline snapshot 与 repo-id 三种 model resolution 下断言 Ref2VA 都走 + `video_reference` 且 ffprobe 为 H.264/H.265；final diff 没有为新增 re-encode/model-routing helper + 增加单测，PR 所报 77 个 video-server tests 只是既有 server/API suite。性能 gate 若启用需另 + 定义 metric directionality。 + ^[PR #5836] + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 From c493e25e2144f81646614768bef05ce3c6c8f792 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 17:17:35 +0800 Subject: [PATCH 044/465] knowledge(vllm-omni): sync 6ec90330 (PR #5165) Upstream-Commit: 6ec90330655b929175ecd7d3b8ea26bcaf87a0a2 Upstream-PR: #5165 Knowledge-Delta: MiniCPM-o vision FlashAttention metadata reuse --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/minicpm-o-4-5/rules.md | 31 ++++++++++++++++++- 10 files changed, 41 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5e81375df..d1118dd9b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 62bf8d630d98a231924f6219b9f53e2ce2879d5b + previous_audited_sha: a874b8e09bb5251bb16e3de8bc22edf3880eec09 audited_ref: main - audited_sha: a874b8e09bb5251bb16e3de8bc22edf3880eec09 + audited_sha: 6ec90330655b929175ecd7d3b8ea26bcaf87a0a2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index bf8de3b4c..6ed15f15b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - a874b8e09bb5251bb16e3de8bc22edf3880eec09`. The canonical machine baseline + 6ec90330655b929175ecd7d3b8ea26bcaf87a0a2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7f1094af5..6e04d4c3c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a874b8e0` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6ec90330` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1ab8dcd22..c40b90a02 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a874b8e0` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6ec90330` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 58919d4a7..52d9d43e6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a874b8e0` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6ec90330` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index bda3cf318..caf824af4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a874b8e0` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6ec90330` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b2f1bead8..56f24bcd4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a874b8e0` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6ec90330` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index bcc73f9f7..dd5b9015d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a874b8e0` 验证存在 +- 源码校验：以上路径均已在 `main @ 6ec90330` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 111e6b63c..a7c33c8e5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a874b8e0`（2026-08-06）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6ec90330`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to a874b8e0 \ + --to 6ec90330 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 8636f3b3c..22137095c 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5638", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5638", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] confidence: high --- @@ -18,6 +18,7 @@ confidence: high |---|---|---| | MiniCPM-o 4.5、`minicpmo_4_5`、版本识别 | MCPMO-2a | `config/pipeline_registry.py::OMNI_PIPELINES["minicpmo_4_5"]` → `model_executor/models/minicpmo_4_5/pipeline.py`；`model_executor/models/registry.py::_OMNI_MODELS` | | Whisper/APM、chunk attention mask、left context/lookahead | MCPMO-2b | `minicpmo_4_5_omni_llm.py::{get_audio_hidden_states,subsequent_chunk_mask}` → audio encoder；row-by-row oracle test | +| SigLIP、FlashAttention、padding/unpad metadata、varlen vision | MCPMO-2c | `SiglipEncoder.forward` → `SiglipEncoderLayer` → `SiglipFlashAttention2._upad_input` | | tokenizer/processor、`trust_remote_code` | MCPMO-1a | pipeline/config factory → `model_executor/models/minicpmo_4_5/` loader | | TTS extra、backend 初始化、空音频 | MCPMO-1b | `minicpmo_4_5_omni_tts.py::MiniCPMO45OmniTTSForConditionalGeneration`、`minicpmo_4_5_token2wav.py::MiniCPMO45Token2wav` | | Code2Wav TensorRT、DiT/Campplus、engine cache/profile | MCPMO-1c | `minicpmo_4_5_code2wav.py::MiniCPMO45Code2Wav` → `batched_token2wav.py::BatchedToken2Wav._estimator_step`；共享实现 `step_audio2/step_audio2_dit_trt.py` | @@ -85,6 +86,34 @@ confidence: high 证据来自 `ee33954dff27da317be597449a6c1b5a5df4052b`，不是 merge target `9235b0ae` 的复测， 因而只能作为外部局部性能证据。^[PR #5382] +## MCPMO-2c — vision unpadding metadata 只在同一次 encoder forward 内复用 + +- 触发：修改 MiniCPM-o 4.5 SigLIP FlashAttention mask、`_get_unpad_data`、Q/K/V layout 或 + encoder layer 调用链。 +- 强制：只有 FlashAttention2 且 `attention_mask is not None` 时，`SiglipEncoder.forward` 才从 + 当前 mask 计算一次 `(indices, cu_seqlens, max_seqlen)`，并把同一 tuple identity 传给该次 + forward 的所有 encoder layers。下一次 forward 必须从新 mask 重算；不得做 global、实例级或 + cross-request cache。直接调用 `SiglipFlashAttention2` 且未传 metadata 时，`_upad_input` 保留 + per-call fallback；dense mask=None path 不计算 metadata。显式传入的 metadata 不会校验是否与 + `attention_mask` 匹配，因此 custom/direct caller 必须成对持有；安全 owner 是 encoder 当前 + forward 的 mask，禁止外部或 stale cache。 +- 强制：metadata 复用必须保持 padded batch 输出与逐 sample unpadded、以及逐 layer 重算 path + 数值一致。Q/K/V projection 的 view→transpose→transpose 原本就形成共享 projection storage 的 + contiguous BSHD；本 PR 没改 layout，新 storage/stride test 只是冻结该既有 zero-copy 合同， + 不能把它计入 metadata 优化收益。 +- 禁止：把 mask shape 相同当成内容相同而跨 forward 复用；将该局部优化泛化成 dense image、 + equal-length video、端到端 serving 或 throughput 提升。review 中短暂加入的 profiler scripts/raw + JSON 已在 final commit history 删除，不是仓库内可复现 benchmark fixture。 +- 验收：CPU spy 覆盖 padded 一次、dense 零次、不同 mask 重算与 tuple identity；L4/FA2 BF16 + 覆盖 encoder reuse 对 per-layer recompute bitwise parity、unpadded 对 eager、两种 padding 程度 + 对逐 sample unpadded（容差 2e-2）。PR 的 2×RTX PRO 6000 Blackwell、27 layers、5 warmups、 + 30 paired samples 中 varlen 约 21.697→13.790 ms / 22.162→14.113 ms，只绑定测试 commit + `895cb303`、该环境与 vision-encoder latency；作者说明 rebase 后 source-file hash 未变，但它 + 仍不是 merge target 的 end-to-end 复测。dense/equal-length video 不计算 metadata且近零变化。 + 当前测试未覆盖 `gradient_checkpointing and training`：新 tuple（两 tensor + Python int）会作为 + 位置参数穿过 `_gradient_checkpointing_func`，补该 path 前不能假设训练/checkpoint 兼容。 + ^[PR #5165] + ## MCPMO-3a — TTS stage 的 batch 能力与 runtime_info 消费一致 - 触发：talker/TTS stage 的 `max_num_seqs`、batching 或 runtime bridge。 From bafe30cf3fa9c397713a6d4514bb8396c08e6cd2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 17:24:08 +0800 Subject: [PATCH 045/465] knowledge(vllm-omni): sync 3d7fc3b9 (PR #5678) Upstream-Commit: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc Upstream-PR: #5678 Knowledge-Delta: typed stage engine-argument projection boundary --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/architecture.md | 9 +++++++- .../components/configuration/rules.md | 22 ++++++++++++++++--- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 11 files changed, 38 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d1118dd9b..ba7651b2f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a874b8e09bb5251bb16e3de8bc22edf3880eec09 + previous_audited_sha: 6ec90330655b929175ecd7d3b8ea26bcaf87a0a2 audited_ref: main - audited_sha: 6ec90330655b929175ecd7d3b8ea26bcaf87a0a2 + audited_sha: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6ed15f15b..62a6596e8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 6ec90330655b929175ecd7d3b8ea26bcaf87a0a2`. The canonical machine baseline + 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6e04d4c3c..977dff891 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6ec90330` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3d7fc3b9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/architecture.md b/knowledge/repos/vllm-omni/components/configuration/architecture.md index 14d8a1ff5..47efd0ed7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/architecture.md +++ b/knowledge/repos/vllm-omni/components/configuration/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: architecture tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #5647", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/diffusion/data.py, vllm_omni/entrypoints/cli/serve.py] +sources: ["claude-workflow-starter-private@296ea45", "PR #5647", "PR #5678", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/diffusion/data.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/entrypoints/cli/serve.py] --- # vLLM-Omni 配置构造架构 @@ -59,6 +59,13 @@ direct kwargs ---------------------------------------> direct normalize + strict structured 与 legacy 可以有不同的最终对象，但不能有不同的字段语义。它们必须在第一位 consumer 前共享 normalization 和 full-key validation；差异只允许发生在校验之后的 typed projection 或对象构造。 +`main @ 3d7fc3b9` 的运行边界仍是过渡态：`VllmOmniConfig` 已能构造 typed stage，且 +`build_engine_args_dict_from_omni_stage_config()` 可将它投影为 backend flat args；但生产 +`StageRuntime`、headless serve 和 diffusion startup 仍把 legacy OmegaConf stage 交给稳定的 +`build_engine_args_dict()`，后者委托 `build_legacy_engine_args_dict()`。因此 typed projection 是 +cutover 前的兼容适配器和测试 oracle，不是当前 live startup source of truth；目标 pin 中 +typed builder 与 `StageConfigFactory.create_from_model` 都没有 non-test caller。 + ## 值状态合同 | 状态 | 必须怎样处理 | diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index a89d5e806..c43422f68 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-08-05 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/engine/stage_init_utils.py, tests/engine/test_stage_engine_args.py] --- # vLLM-Omni 配置开发门禁 @@ -20,7 +20,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "zu | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| -| strict schema、unknown field、alias、flat→nested、structured/legacy/direct parity、typed projection | `strict-normalization`：`VOMNI-CFG-1a`–`1g` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,strip_parent_engine_args}` → `vllm_omni/config/omni_config.py::{_build_diffusion_config_projection,VllmOmniConfig.from_pipeline_config}` → `vllm_omni/diffusion/data.py::{normalize_omni_diffusion_kwargs,OmniDiffusionConfig.from_kwargs}` | +| strict schema、unknown field、alias、flat→nested、structured/legacy/direct parity、typed projection | `strict-normalization`：`VOMNI-CFG-1a`–`1h` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,strip_parent_engine_args}` → `vllm_omni/config/omni_config.py::{_build_diffusion_config_projection,VllmOmniConfig.from_pipeline_config}` → `vllm_omni/engine/stage_init_utils.py::{build_engine_args_dict,build_engine_args_dict_from_omni_stage_config}` | | deploy YAML、`base_config`、pipeline/stage overlay、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a` | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::{StageConfigFactory.create_from_model,StageConfigFactory._merge_cli_overrides}` | | composable strategy、axis、routing、load balancing、`strategy-config` | `composable-strategy`：`CONF-4a` | `vllm_omni/config/composable_parallel/strategy_loader.py::{parse_strategy_specs,load_strategy_specs}` → `translator.py::translate_strategy_stack` → `apply.py::apply_strategy_specs` → `config_factory.py::{StageConfigFactory._apply_strategy_specs,StageConfigFactory._reconcile_strategy_with_cli}` | | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | @@ -28,7 +28,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "zu | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| | `core` | 每次配置审查 | `VOMNI-CFG-1b`, `VOMNI-CFG-1c` | -| `strict-normalization` | schema、alias、unknown field、structured/legacy/direct 路径 | `VOMNI-CFG-1a`, `VOMNI-CFG-1b`, `VOMNI-CFG-1c`, `VOMNI-CFG-1d`, `VOMNI-CFG-1e`, `VOMNI-CFG-1f`, `VOMNI-CFG-1g` | +| `strict-normalization` | schema、alias、unknown field、structured/legacy/direct 路径 | `VOMNI-CFG-1a`, `VOMNI-CFG-1b`, `VOMNI-CFG-1c`, `VOMNI-CFG-1d`, `VOMNI-CFG-1e`, `VOMNI-CFG-1f`, `VOMNI-CFG-1g`, `VOMNI-CFG-1h` | | `deploy-memory` | 显存预算、KV pin、多 stage 共卡 | `CONF-1a`, `CONF-2a` | | `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`, `CONF-4b`, `CONF-5a` | | `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a` | @@ -43,6 +43,22 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "zu - **VOMNI-CFG-1e — 严格配置固定按“归一化、验 owner、分区、构造”执行。** deploy/stage/CLI overlay、alias 和 flat→nested 转换必须先产出一份规范化映射；mixed CLI/orchestrator namespace 只允许先路由已明确声明的非 stage scope，剩余 stage 候选必须以完整 key 集合进入 ownership validator。validator 不能同时改值、路由字段或先丢 `None`；校验通过后的规范化结果就是后续分区和构造唯一可消费的 owner artifact，禁止只调用 validator 却继续传 raw mapping，也禁止末端重新做 alias、default、precedence 或静默过滤。structured、legacy startup 和仍公开可调用的 direct factory 必须复用同一组 normalization/validation primitive。 - **VOMNI-CFG-1f — key 是否已知与 value 是否有效必须分开判定。** 缺失、已知字段的 `None`、显式 `False`/`0`、未知字段的 `None` 和两个非空来源冲突是五种不同状态：只有明确 owner 可以把已知 `None` 解释为 unset，`False`/`0` 不得被 truthiness merge 吃掉，未知 key 即使为 `None` 也必须按 strict contract 报错，alias 只在 legacy 与 canonical 都提供非 `None` 值时冲突。验收至少覆盖本轮会改变分支的这些状态，并断言错误或最终 consumer。 - **VOMNI-CFG-1g — rebase 或 schema 扩张后重新做配置归属审计。** 上游新增 dataclass 字段、默认 factory 参数、pipeline-wide 字段或 owner 集合变化时，把自动合并结果当作新的语义 diff；重新比较 base/head 的字段集合、序列化边界、默认-stage 透传和所有严格入口。默认 factory 的扩展字段应由 schema metadata 或单一声明集合驱动，禁止再加一份手写白名单。只有最新 base 上的真实 factory/legacy/structured 路径都保留合法非默认值并继续拒绝未知字段，才算 rebase 完成。 +- **VOMNI-CFG-1h — typed engine projection 与生产 cutover 是两个 merge condition。** + `build_engine_args_dict_from_omni_stage_config` 从 model/load/cache/scheduler/runtime/connector/ + parallel/quantization/diffusion typed owner 深拷贝非 `None` 字段，再与 legacy 共用 model/tokenizer + precedence、connector injection、worker resolution、generation cache defaults 与 diffusion attention + normalization；不允许增加 catch-all `engine_args`。但在 target `3d7fc3b9`，稳定入口 + `build_engine_args_dict` 仍明确委托 `build_legacy_engine_args_dict`，runtime/headless/diffusion startup + 仍传 legacy stage shape；`StageConfigFactory.create_from_model` 与 + `build_engine_args_dict_from_omni_stage_config` 均无 non-test caller，所以 typed path 是纯 + preparatory/test-only，不能声称生产已经“从 VllmOmniConfig 读取”。 + cutover 前须把 strategy/replica/startup-plan 一并改为 typed input，并用 live startup 测试证明。 + review 暴露的 typed-only defaults、unowned field 静默丢失、model/tokenizer subdir 与 omni-KV + precedence 问题，final 以“保留 unset backend default、合并后按 execution type 验 owner、未知 + owner 拒绝、全 backend field 双向 effective parity”修正；新增字段后该 exhaustive census 必须 + 重新运行，不能只维护几个 deferred-default 白名单。但 registry parity test 会跳过没有具体 HF + config 就无法 `resolve_pipeline_config` 的 callable entry（当前包括 `qwen3_omni_moe`），并非字面 + exhaustive；cutover 前每种 resolver variant 都须补 concrete HF-config fixture。^[PR #5678] ## 部署配置与资源预算 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c40b90a02..ff2a383e2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6ec90330` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 3d7fc3b9` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 52d9d43e6..483fc8f00 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6ec90330` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3d7fc3b9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index caf824af4..900943ca4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6ec90330` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3d7fc3b9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 56f24bcd4..76e062af9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6ec90330` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3d7fc3b9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index dd5b9015d..2fcff0570 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6ec90330` 验证存在 +- 源码校验：以上路径均已在 `main @ 3d7fc3b9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a7c33c8e5..3f6c82b8d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6ec90330`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3d7fc3b9`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 6ec90330 \ + --to 3d7fc3b9 \ --repo \ --mode report-only ``` From 07ff93e5204bf0a214e53b76ace6a25238f0024f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 17:30:37 +0800 Subject: [PATCH 046/465] knowledge(vllm-omni): sync 24ede9e2 (PR #5255) Upstream-Commit: 24ede9e226a62910455d5d8da4c71d5c5177c292 Upstream-PR: #5255 Knowledge-Delta: Ring SDPA GQA and MQA head-layout contract --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 32 +++++++++++++++++-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- 11 files changed, 41 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ba7651b2f..cc083f0a3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6ec90330655b929175ecd7d3b8ea26bcaf87a0a2 + previous_audited_sha: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc audited_ref: main - audited_sha: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc + audited_sha: 24ede9e226a62910455d5d8da4c71d5c5177c292 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 62a6596e8..f964d97f0 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc`. The canonical machine baseline + 24ede9e226a62910455d5d8da4c71d5c5177c292`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 230aec0d7..249cc98c7 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 977dff891..1962d8e38 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3d7fc3b9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 24ede9e2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ff2a383e2..0b2a1c744 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3d7fc3b9` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 24ede9e2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 304433180..74777cfb0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5839", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5839", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] confidence: high --- @@ -29,7 +29,7 @@ confidence: high | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | | modular/multi-DiT、`_dit_modules`、Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f` | pipeline runtime component list → registry/runner/cache/LoRA/module discovery consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | -| paged KV/cache、backend/platform、GQA/layout、预算与 admission | `system-runtime`：`DIFF-4a`–`4f` | engine init → `diffusion_kv/{initialization,paged_attention_adapter}.py` → platform hook → scheduler | +| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、预算与 admission | `system-runtime`：`DIFF-4a`–`4g` | engine init → attention parallel/backend → platform hook → scheduler | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | | 审查组 | 什么时候触发 | 规则 ID | @@ -38,7 +38,7 @@ confidence: high | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、async output readiness | `DIFF-1a`–`1e` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT lifecycle | `DIFF-2a`–`2f` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4f` | +| `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4g` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | ## 优化路径与 eager 的等价合同 @@ -290,5 +290,31 @@ confidence: high - 验收：32Q/8KV、异构 prefix、单 token/单 head、非对称等维 LSE 和 batch compaction 精确 对照 dense/reference 输出。 ^[PR #5543] ^[PR #6102] +### DIFF-4g — Ring GQA 只在通信后扩展，Hybrid Ulysses 不把 padding 当真 head + +- 触发：修改 Ring SDPA、GQA/MQA head 数、LSE accumulation、Ulysses all-to-all 或 + `advanced_uaa` head padding。 +- 强制：Ring 始终以原始 `H_kv` 通信 K/V；仅在本 rank 的 aten SDPA 调用前，当 + `H_q != H_kv` 时要求 `H_q % H_kv == 0`，再用 `repeat_interleave(H_q/H_kv)` 将 K/V 扩成 + Q head 数。该 local materialization 增加 kernel 输入计算/显存，但不增加 ring 通信量；不能 + 提前到 send/recv 前。之所以不能直接用 `enable_gqa`，是该 path 需要 aten flash/efficient + SDPA 返回 LSE 供 ring accumulation，而这些调用没有该参数。实现以 K head 数计算 factor 并 + 同样 repeat V，却未显式检查 `H_v == H_k`；现有测试始终令两者相等，future caller/backend 必须 + 保持或提前断言该 invariant，不能等 SDPA 在更深处 cryptic failure。 +- 强制：`advanced_uaa` + Hybrid Ulysses/ Ring 时，在 Ulysses all-to-all 前检查 key 和 value + 原始 head 数均可被 `ulysses_degree` 整除；否则 fail-fast。padding 出来的 zero K/V head 若交给 + Ring 再 replicate 会被当成真实 group，不能作为合法 MQA/GQA。pure Ring（Ulysses=1）仍支持 + 1 或多 KV heads；valid Hybrid positive fixture 必须使用可整除布局。 +- 禁止：用默认 backend 的绿测证明 SDPA 修复（安装 FA/FA3 时会绕过）；用 num_heads=3、 + Ulysses=2 的 padded case 当 Hybrid 正向测试；从通信量不变推断端到端性能不变。 +- 验收：4-card test 显式设 `TORCH_SDPA`，以 single-rank baseline 对 pure Ring GQA(8Q/2KV)、 + MQA(8Q/1KV) 和既有 Ulysses+Ring/AllGather-KV 输出，BF16 容差 5e-2；CPU guard 覆盖 Hybrid + padded-MQA 拒绝，另以 4Q heads、Ulysses2×Ring2 验证 valid Hybrid。ready CI 的 L4×4 job + 已收集 `test_attention_sp.py`，但不收集 `test_ulysses_uaa.py`；PR comment 的 primary matrix + 是 pre-final head `e6857888` 在 4×B300 上 4 passed、max abs 0.015625，runtime fix 已在但 final + target 未复测。修正为 4 heads 的 Hybrid 25.30s pass 只见 final PR body，未附 hardware/SHA/log。 + 因此目标 pin 有局部数值证据和未来 L4 gate wiring，但没有附带 L4 run 产物，Hybrid + guard/positive 也未被该新 job 覆盖。^[PR #5255] + 相关执行流见 [Diffusion architecture](architecture.md)；benchmark 证据合同见 [performance evidence](../../benchmark/guides/performance-evidence.md)。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 483fc8f00..25ddf8464 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3d7fc3b9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 24ede9e2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 900943ca4..824525ef3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3d7fc3b9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 24ede9e2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 76e062af9..95119a5fc 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3d7fc3b9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 24ede9e2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2fcff0570..49691efe6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3d7fc3b9` 验证存在 +- 源码校验：以上路径均已在 `main @ 24ede9e2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3f6c82b8d..02ea4e76e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3d7fc3b9`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 24ede9e2`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 3d7fc3b9 \ + --to 24ede9e2 \ --repo \ --mode report-only ``` From 740cc36eb4dab42d556b2a5fe1e3e152e439a75a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 17:36:15 +0800 Subject: [PATCH 047/465] knowledge(vllm-omni): sync ab459b36 (PR #5344) Upstream-Commit: ab459b36c26d5c7903f623156d88ab2ae06f0a97 Upstream-PR: #5344 Knowledge-Delta: FlashInfer mixed-dtype attention plan lifecycle --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 33 +++++++++++++++++-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- 10 files changed, 41 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cc083f0a3..8962f55d8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc + previous_audited_sha: 24ede9e226a62910455d5d8da4c71d5c5177c292 audited_ref: main - audited_sha: 24ede9e226a62910455d5d8da4c71d5c5177c292 + audited_sha: ab459b36c26d5c7903f623156d88ab2ae06f0a97 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f964d97f0..5795fd6c1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 24ede9e226a62910455d5d8da4c71d5c5177c292`. The canonical machine baseline + ab459b36c26d5c7903f623156d88ab2ae06f0a97`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1962d8e38..796e75f7e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 24ede9e2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ab459b36` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0b2a1c744..65376b6ec 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 24ede9e2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ab459b36` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 74777cfb0..be90208f7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5839", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5839", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] confidence: high --- @@ -29,7 +29,7 @@ confidence: high | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | | modular/multi-DiT、`_dit_modules`、Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f` | pipeline runtime component list → registry/runner/cache/LoRA/module discovery consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | -| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、预算与 admission | `system-runtime`：`DIFF-4a`–`4g` | engine init → attention parallel/backend → platform hook → scheduler | +| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4h` | engine init → attention parallel/backend → platform hook → scheduler | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | | 审查组 | 什么时候触发 | 规则 ID | @@ -38,7 +38,7 @@ confidence: high | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、async output readiness | `DIFF-1a`–`1e` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT lifecycle | `DIFF-2a`–`2f` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4g` | +| `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4h` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | ## 优化路径与 eager 的等价合同 @@ -316,5 +316,32 @@ confidence: high 因此目标 pin 有局部数值证据和未来 L4 gate wiring，但没有附带 L4 run 产物，Hybrid guard/positive 也未被该新 job 覆盖。^[PR #5255] +### DIFF-4h — FlashInfer mixed-dtype plan 必须绑定完整 runtime shape 与 mask 内容 + +- 触发：修改 `FLASHINFER_ATTN`、`BatchPrefillWithRaggedKVCacheWrapper`、QK16/V8、backend + selection、ragged indptr/plan cache 或 custom mask fallback。 +- 强制：输入保持初始化时 CUDA device；Q/K 只允许 fp16/bf16 override，V 另允许 + fp8_e4m3，wrapper 输出恢复原 query dtype。auto backend 按 compute capability 选 major>=10 + `cute-dsl`、major>=9 `fa3`、否则 `fa2`；non-CuTe 才接受 custom mask，CuTe nontrivial mask、 + causal+explicit mask 或无法 pack 的 mask 必须回退原始输入的 Torch SDPA。 +- 强制：plan key 至少绑定 batch、Q/KV length、Q/KV heads、QK/K/VO head dim、Q/K/V/output + dtype、causal、scale 与是否有 mask；shape/dtype 改变重建 indptr/plan，unmasked 同 key 可复用。 + mask shape 相同也可能内容不同，所以每次 masked call 都必须重新 plan，不能只按 pointer/shape + cache。Q/K/V cast 留在 compile-visible path，wrapper plan/run 边界才 disable compiler。目标 key + 虽记录 `head_dim_k`，plan 只接收 `head_dim_qk` 与 `head_dim_vo`，且未断言 Q/K head dim 相等； + caller 必须保持 `D_q == D_k`，并补 mismatch fail-fast test，不能把 key 字段误当 backend 已消费。 +- 禁止：把本 PR 早期 Sage/TRTLLM kernels 当成 merged scope（review 后删除并转交 TRTLLM + backend）；把 activation 直接 cast 成 FP8 描述成带 scale 的 checkpoint quantization；用 + `supports_attention_mask=True` 推断 CuTe 支持任意 mask。 +- 验收：mixed QK/V dtype 只能在 FlashInfer >=0.6.16rc1；目标对可解析的 <=0.6.15 fail-fast， + 但没有 dependency pin，只有 docs/code gate。该 gate 比较配置 override：两者均为 None 时立即 + 返回，即使 caller 实际传入 mixed Q/K 与 V tensor 也不拦；capability 必须最终按 plan 中真实 + q/k/v dtype 校验。缺失/非法 `flashinfer.__version__` 也会直接放行，是未封闭 gap。须补测试覆盖 + dtype allowlist、old/missing version、auto backend、device mismatch、plan reuse/replan、同 shape + 不同 mask、CuTe/causal fallback、GQA 与 output dtype；本 PR 没有新增 repo test，只有外部 smoke。 + PR 的 Cosmos3 1280×720、35 steps、seed42，B200/B300 DiT timing 与 Nano 前24帧 LPIPS=0.051496 + 只绑定 vLLM-Omni `4ce23c33` + FlashInfer `d0889c7c` 的 pre-final 环境；final config/version gate + commit 后无复测，且这些不是端到端 latency、稳定 speedup 或通用质量阈值。^[PR #5344] + 相关执行流见 [Diffusion architecture](architecture.md)；benchmark 证据合同见 [performance evidence](../../benchmark/guides/performance-evidence.md)。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 25ddf8464..7190a845c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 24ede9e2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ab459b36` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 824525ef3..d20d35c8b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 24ede9e2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ab459b36` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 95119a5fc..250ea9064 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 24ede9e2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ab459b36` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 49691efe6..6fab33314 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 24ede9e2` 验证存在 +- 源码校验：以上路径均已在 `main @ ab459b36` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 02ea4e76e..b205ee45c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 24ede9e2`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ab459b36`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 24ede9e2 \ + --to ab459b36 \ --repo \ --mode report-only ``` From c02bd593dcd28bb51b1ba4da852631ab6bb40456 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 17:45:01 +0800 Subject: [PATCH 048/465] knowledge(vllm-omni): sync f1141134 (PR #5838) Upstream-Commit: f11411340978a80fbec06d6d7ef389a6d84e697e Upstream-PR: #5838 Knowledge-Delta: diffusion metadata architecture alias resolution --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 26 ++++++++++++++++--- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- 10 files changed, 34 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8962f55d8..18d052ffa 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 24ede9e226a62910455d5d8da4c71d5c5177c292 + previous_audited_sha: ab459b36c26d5c7903f623156d88ab2ae06f0a97 audited_ref: main - audited_sha: ab459b36c26d5c7903f623156d88ab2ae06f0a97 + audited_sha: f11411340978a80fbec06d6d7ef389a6d84e697e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5795fd6c1..8eeac7b7a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - ab459b36c26d5c7903f623156d88ab2ae06f0a97`. The canonical machine baseline + f11411340978a80fbec06d6d7ef389a6d84e697e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 796e75f7e..693e7a0a9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ab459b36` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f1141134` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 65376b6ec..5bd70ed5c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ab459b36` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f1141134` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index be90208f7..1d786d333 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5839", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py] confidence: high --- @@ -29,7 +29,7 @@ confidence: high | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | | modular/multi-DiT、`_dit_modules`、Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f` | pipeline runtime component list → registry/runner/cache/LoRA/module discovery consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | -| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4h` | engine init → attention parallel/backend → platform hook → scheduler | +| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4i` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | | 审查组 | 什么时候触发 | 规则 ID | @@ -38,7 +38,7 @@ confidence: high | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、async output readiness | `DIFF-1a`–`1e` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT lifecycle | `DIFF-2a`–`2f` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、异常与并发 | `DIFF-4a`–`4h` | +| `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4i` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | ## 优化路径与 eager 的等价合同 @@ -343,5 +343,25 @@ confidence: high 只绑定 vLLM-Omni `4ce23c33` + FlashInfer `d0889c7c` 的 pre-final 环境；final config/version gate commit 后无复测，且这些不是端到端 latency、稳定 speedup 或通用质量阈值。^[PR #5344] +### DIFF-4i — 模型能力 metadata 必须跨 architecture 与 pipeline 两个 key space 解析 + +- 触发：修改 diffusion registry、HF `architectures`、`model_class_name` 或共享 + `DiffusionModelMetadata`，尤其是多图数量、mixed-reference 与 attention-mask-free 能力。 +- 强制：先接受 metadata 表中的 direct pipeline key；未命中时只通过 registry 的 + `architecture → pipeline class` 映射解析一次，再查 metadata。unknown architecture、`None`，以及 + registry 映射到没有 metadata 的 pipeline 都保持空 defaults；registry tuple 用解包读取，使布局 + 变化显式失败。此 helper 返回整份 metadata，影响 config、AsyncOmni serving view、视频 mixed + reference 和 attention backend，不得把修改只当作 image-count 局部修复。 +- 禁止：因 architecture 已注册就假定其 pipeline 有 metadata；递归猜测别名；让 direct key 被 + registry 映射覆盖；或把 defaults 的 `supports_multimodal_inputs=false` 当成“数量未知”——image-edit + admission 会把它解释为最多一张图，而 `supports=true, max=None` 才是不设上限。 +- 验收：同时覆盖 direct pipeline key、architecture→metadata-bearing pipeline、unknown/`None`、 + 以及 architecture→无 metadata pipeline；通过 `OmniDiffusionConfig.update_multimodal_support()` + 断言整份能力传播，并在 serving 边界验证 false→1、true+整数上限与无上限行为。PR #5838 只新增 + Qwen direct 与 Hunyuan architecture→pipeline 的手动 config 测试；未执行真实 architecture 解析、 + AsyncOmni 或 HTTP edit 路径，也没有 registry/metadata census 和 negative alias 回归。其 E2E 结果 + 只有截图，缺命令文本日志、vLLM 版本、upstream SHA 与重复次数。helper 仍 function-local import + 私有 `_DIFFUSION_MODELS`，review 提议公开改名但 final 未采用。^[PR #5838] + 相关执行流见 [Diffusion architecture](architecture.md)；benchmark 证据合同见 [performance evidence](../../benchmark/guides/performance-evidence.md)。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7190a845c..19f6bad22 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ab459b36` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f1141134` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index d20d35c8b..3ccd82301 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ab459b36` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f1141134` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 250ea9064..78948c65d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ab459b36` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f1141134` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6fab33314..f6b890fd7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ab459b36` 验证存在 +- 源码校验：以上路径均已在 `main @ f1141134` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b205ee45c..6d74d56ee 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ab459b36`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f1141134`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to ab459b36 \ + --to f1141134 \ --repo \ --mode report-only ``` From 936b9f847d10b5ae604053f5026974e41b54075d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 17:58:34 +0800 Subject: [PATCH 049/465] knowledge(vllm-omni): sync e906f41d (PR #5884) Upstream-Commit: e906f41dc8ea12cecdee85a59b62e1e488253d3b Upstream-PR: #5884 Knowledge-Delta: Cache-DiT dotted component lifecycle --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-component-lifecycle.md | 51 +++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 16 ++---- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/models/bagel/rules.md | 23 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 4 +- 12 files changed, 89 insertions(+), 24 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 18d052ffa..e6bb0c5f3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ab459b36c26d5c7903f623156d88ab2ae06f0a97 + previous_audited_sha: f11411340978a80fbec06d6d7ef389a6d84e697e audited_ref: main - audited_sha: f11411340978a80fbec06d6d7ef389a6d84e697e + audited_sha: e906f41dc8ea12cecdee85a59b62e1e488253d3b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8eeac7b7a..218265b41 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - f11411340978a80fbec06d6d7ef389a6d84e697e`. The canonical machine baseline + e906f41dc8ea12cecdee85a59b62e1e488253d3b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 693e7a0a9..f8c271530 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f1141134` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e906f41d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5bd70ed5c..d6984282d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f1141134` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e906f41d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -30,6 +30,7 @@ sources: [] |---|---| | 理解共享职责和数据流 | [architecture](architecture.md) | | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | +| multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md new file mode 100644 index 000000000..c9e824ba8 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md @@ -0,0 +1,51 @@ +--- +title: "Diffusion component lifecycle 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #5720", "PR #5884", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py] +confidence: high +--- + +# Diffusion component lifecycle 规则 + +本页承载共享 diffusion owner 的 multi-DiT component discovery 合同；模型专有 component 名和 +residency 留在模型 owner。规则入口与其他共享机制仍见 [Diffusion 共享规则](rules.md)。 + +### DIFF-2f — 多 DiT pipeline 的 component list 是所有共享 lifecycle 的唯一集合 + +- 触发：pipeline 增加第二个 DiT、task-dependent transformer，或修改 `_dit_modules`。 +- 强制：registry SP injection、loader strictness、Cache-DiT enable/refresh/summary、regional + compile、LoRA、offloader/module collection 都遍历运行时实际存在的 `_dit_modules`；允许实例按 + task 收窄 list，不得由 class 默认预建不存在的组件。模型规则只补 task/权重选择与 residency。 +- 禁止：硬编码 `transformer`/`transformer_2` 或把非 canonical attribute 静默漏掉；不能用 + 第一个 DiT 成功推断第二个已启用 cache/compile/offload。 +- 验收：用非 canonical 第二属性覆盖每个 consumer 的 enable/refresh/summary/discovery，并覆盖 + task 收窄后的缺省属性；至少一个真实双 DiT pipeline 逐 route 验证。^[PR #5720] + +### DIFF-2g — dotted component path 必须跨 consumer 解析为稳定且唯一的 module identity + +- 触发：`_dit_modules` 增加含点 path、alias，或 consumer 新增 discovery/getter/mutation。 +- 强制：把名称视为逐段 attribute path；中间 attribute 缺失或最终值为 `None` 才算未实例化。 + discovery、enable、refresh、compile、SP、LoRA 与 offload 必须解析到同一 object；结果按 identity + 去重或明确禁止 alias，lifecycle 内不得悄悄替换已解析 target。 +- 禁止：把含点字符串传给单层 `getattr`/`hasattr`；依赖 conventional direct alias 代替 canonical + declaration；同一 object 因重复 path 被 enable 两次。目标 pin 只有 Cache-DiT 与 offloader 使用 + `attrgetter`，generic compile、registry SP 与 LoRA 仍未统一，因此不能宣称 cross-consumer parity。 +- 验收：flat、dotted、missing-middle、`None`、非字符串、重复 alias 与 enable 后 identity replacement + 分别覆盖所有 consumer。当前 mock 只覆盖 Cache-DiT dotted enable/refresh/summary，没有真实 + cross-consumer identity 或异常覆盖。^[PR #5884] + +### DIFF-2h — Cache-DiT summary 只汇报 active cached target + +- 触发：修改 Cache-DiT summary、custom enabler、`BlockAdapter.is_cached` 或声明包含 uncached DiT。 +- 强制：discovery 与 summary 使用同一 path 解析，但调用 `cache_dit.summary` 前按真实 active cache + context 过滤；必须定义全部 discovered target 都 uncached 时的可观察结果。Cosmos3 等 pipeline + 可声明只执行一次而刻意不缓存的 nested DiT，声明存在不等于 cache 已启用。 +- 禁止：对所有 declared DiT 无条件 summary；依赖 “no transformer” warning 判断 active cache。 + 当前实现对 all-uncached 情况静默无 summary、也无 warning，调用方必须接受或另加显式信号。 +- 验收：混合 cached/uncached、all-uncached、无 declared target 和 repeated summary 分别断言调用与 + 日志。当前 mock 覆盖混合列表只汇报 cached outer target；没有真实 exception 或 teardown 证据。 + 目标 `CacheBackend` 只有 enable/refresh/is_enabled；future 若增加 shutdown/teardown，必须清理同一 + resolved target。^[PR #5884] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 1d786d333..2c24afda3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -27,7 +27,7 @@ confidence: high | HSDP/FSDP、`fully_shard`、DeviceMesh、packed/scalar parameter、FP8 | `checkpoint-distributed`：`DIFF-2b` | `distributed/hsdp.py::{apply_hsdp_to_model,shard_model}` → loader `_load_model_with_hsdp` → `hsdp_fp8.py` | | distributed layerwise offload、AllGather、异构 block、shared buffer | `checkpoint-distributed`：`DIFF-2e` | `offloader/distributed_layerwise_backend.py::{DistributedLayerwiseOffloadHook.initialize_hook,prefetch_layer,DistributedLayerwiseOffloadBackend._allocate_shared_buffers}` | | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | -| modular/multi-DiT、`_dit_modules`、Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f` | pipeline runtime component list → registry/runner/cache/LoRA/module discovery consumers | +| modular/multi-DiT、`_dit_modules`、Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2h` | pipeline runtime component list → registry/runner/cache/LoRA/module discovery consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4i` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | @@ -36,7 +36,7 @@ confidence: high |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、async output readiness | `DIFF-1a`–`1e` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT lifecycle | `DIFF-2a`–`2f` | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT lifecycle | `DIFF-2a`–`2h` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4i` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | @@ -198,16 +198,8 @@ confidence: high final repo-root modular route 未复测，仍不得升级为 production support。TP+no-AllGather、一般 DP+SP 等其他组合仍缺完整 model×hardware E2E。^[PR #5764] ^[PR #5839] ^[PR #5836] -### DIFF-2f — 多 DiT pipeline 的 component list 是所有共享 lifecycle 的唯一集合 - -- 触发：pipeline 增加第二个 DiT、task-dependent transformer，或修改 `_dit_modules`。 -- 强制：registry SP injection、loader strictness、Cache-DiT enable/refresh/summary、regional - compile、LoRA、offloader/module collection 都遍历运行时实际存在的 `_dit_modules`；允许实例按 - task 收窄 list，不得由 class 默认预建不存在的组件。模型规则只补 task/权重选择与 residency。 -- 禁止：硬编码 `transformer`/`transformer_2` 或把非 canonical attribute 静默漏掉；不能用 - 第一个 DiT 成功推断第二个已启用 cache/compile/offload。 -- 验收：用非 canonical 第二属性覆盖每个 consumer 的 enable/refresh/summary/discovery，并覆盖 - task 收窄后的缺省属性；至少一个真实双 DiT pipeline 逐 route 验证。^[PR #5720] +多 DiT component discovery 与 dotted-path lifecycle 合同见 +[DIFF-2f–2h 专页](rules-component-lifecycle.md)。 ## 质量阈值与资源辅助 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 19f6bad22..8e5ffd78d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f1141134` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e906f41d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3ccd82301..c9f552627 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f1141134` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e906f41d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 78948c65d..dc3f165e6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f1141134` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e906f41d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f6b890fd7..cdca0718f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f1141134` 验证存在 +- 源码校验：以上路径均已在 `main @ e906f41d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/bagel/rules.md b/knowledge/repos/vllm-omni/models/bagel/rules.md index 77b8061b5..279db1750 100644 --- a/knowledge/repos/vllm-omni/models/bagel/rules.md +++ b/knowledge/repos/vllm-omni/models/bagel/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5775", vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/lance/lance_transformer.py] +sources: ["PR #5775", "PR #5884", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/diffusion/models/lance/lance_transformer.py, tests/diffusion/cache/test_cache_backends.py] confidence: high --- @@ -29,3 +29,24 @@ batch/branch 关系由重复 query sequence、`packed_seqlens` 和 merged cache - 保留 BAGEL 1-D 路径的回归覆盖，并为 Lance `(3,S)` 路径补 GPU E2E。引入该合同的 PR #5775（`c9f2e5ad`）只有 shape tracing、ruff 和 reviewer approval，作者因无 CUDA GPU 未执行 Lance 端到端，因此不能把该提交视为 GPU 精度/功能已验证。 + +## BAGEL-2：DiT owner 是 nested `language_model.model` + +**适用范围**：任何消费 `BagelPipeline._dit_modules` 的 Cache-DiT、compile、SP、LoRA、loader 或 +offloader 路径；Lance 继承此 pipeline 时也需复核其实际组件树。 + +**合同**：Bagel 声明 `_dit_modules = ["language_model.model"]`；consumer 必须把它当多段 attribute +path 解析到真实 DiT，不能把含点字符串传给单层 `getattr`。缺失中间节点应按该 consumer 的既有 +策略 skip/warn/fail，不能因解析器差异静默漏掉模型。目标 pin 的 Cache-DiT 已支持 dotted +enable/refresh/summary；generic compile、registry SP 和 LoRA 仍未共享 dotted resolver，不能声称 +Bagel 的全部 lifecycle 已支持。 +Bagel init 也令 `transformer` 指向同一 object，但 `_dit_modules` 非空时 Cache-DiT 不走 conventional +fallback；consumer 必须以 canonical declaration 为准，不能依赖当前 direct alias。Cache-DiT 会在 +refresh/summary 时重新解析该 path，故 enable 后不得替换 `language_model.model` identity；若 future +declaration 同时列出两个 alias，也会重复 enable，因为当前 discovery 不按 identity 去重。 + +**验收**：真实 Bagel component tree 覆盖 Cache-DiT enable→repeated request refresh→summary，断言 +hook/context 始终绑定同一 nested model；目标 cache backend 没有 disable API，future 若增加 worker +shutdown/teardown 必须验证清理同一 target。对 compile、SP、LoRA 与 offload 分别验证同一 object +identity 和 missing-middle 行为。当前 mock 测试只覆盖 Cache-DiT dotted enable/refresh/summary， +不构成真实 cross-consumer 或 teardown 证据。^[PR #5884] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6d74d56ee..0f7443ac1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f1141134`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e906f41d`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to f1141134 \ + --to e906f41d \ --repo \ --mode report-only ``` From 3d434ca3be661a3adc42080ed9a1bd660d0f9f5d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 18:10:33 +0800 Subject: [PATCH 050/465] knowledge(vllm-omni): sync d8c35663 (PR #5853) Upstream-Commit: d8c35663b47e54346062e6eac312e7b65bbb6829 Upstream-PR: #5853 Knowledge-Delta: request-scoped MiniMax H3 Cache-DiT quality policy --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-component-lifecycle.md | 36 +++++++++++++++++-- .../vllm-omni/components/diffusion/rules.md | 6 ++-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/models/bagel/rules.md | 8 ++--- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/minimax-h3/rules.md | 36 ++++++++++++++++++- 13 files changed, 86 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e6bb0c5f3..6899d23ad 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f11411340978a80fbec06d6d7ef389a6d84e697e + previous_audited_sha: e906f41dc8ea12cecdee85a59b62e1e488253d3b audited_ref: main - audited_sha: e906f41dc8ea12cecdee85a59b62e1e488253d3b + audited_sha: d8c35663b47e54346062e6eac312e7b65bbb6829 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 218265b41..6a34e3518 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - e906f41dc8ea12cecdee85a59b62e1e488253d3b`. The canonical machine baseline + d8c35663b47e54346062e6eac312e7b65bbb6829`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f8c271530..041315b80 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e906f41d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d8c35663` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d6984282d..38c2762f6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e906f41d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d8c35663` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md index c9e824ba8..b025068d4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5720", "PR #5884", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py] +sources: ["PR #5720", "PR #5853", "PR #5884", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py] confidence: high --- @@ -47,5 +47,35 @@ residency 留在模型 owner。规则入口与其他共享机制仍见 [Diffusio 当前实现对 all-uncached 情况静默无 summary、也无 warning，调用方必须接受或另加显式信号。 - 验收：混合 cached/uncached、all-uncached、无 declared target 和 repeated summary 分别断言调用与 日志。当前 mock 覆盖混合列表只汇报 cached outer target；没有真实 exception 或 teardown 证据。 - 目标 `CacheBackend` 只有 enable/refresh/is_enabled；future 若增加 shutdown/teardown，必须清理同一 - resolved target。^[PR #5884] + request-scoped runtime 已增加 disable，但 summary 测试没有覆盖 disable 后状态。^[PR #5884] + +### DIFF-2i — request-scoped Cache-DiT 状态迁移必须保持唯一、可恢复的 ownership + +- 触发：pipeline 接管 runner 启动的 Cache-DiT，或 request 字段选择 cache profile/reference path。 +- 强制：只有实现 opt-in protocol 的 pipeline 才接管已 enabled backend；runner 转移 ownership 后 + 清空自己的引用。每个 request 在 denoise 紧邻边界解析 desired spec：同 installation key 只 refresh， + key 改变先 disable 旧 hooks 再 enable/refresh 新 profile，`None` disable 且 repeated `None` no-op； + disable/enable 失败时必须保留尚未清理的 target 或 rollback 已安装 hooks，使下一次 transition 能恢复。 +- 禁止：runner 与 pipeline 同时拥有同一 backend；按 config object equality 代替 stable installation + key；enable 前覆盖既有 target list；第一个 disable exception 后忘记其余 target；在 cleanup 未完成时 + 仅清 runtime key/reference 并宣称 disabled。非 opt-in 模型继续使用 runner-owned lifecycle，其他 + cache backend 不变。 +- 验收：覆盖 startup adoption、omit→lossless→accelerated→omit、无 startup 的 accelerated→lossless、 + same-key repeated refresh、different-key reinstall、partial multi-target enable、每个位置的 disable + exception、取消与 worker shutdown，并断言 policy prepare 严格先于 denoise。当前实现的正常迁移有 + fake backend 覆盖；但 enable 可留下未被 runtime 接管的部分 hooks，disable 在首个 exception 后仍在 + `finally` 丢弃全部 target/state，因此不能称为 failure-atomic，也没有真实 hook 的并发、rollback、取消 + 或 shutdown 证据。^[PR #5853] + +### DIFF-2j — process-wide request policy 必须参与 batch compatibility identity + +- 触发：request 字段或模型 policy 会切换 worker-wide hook、adapter、compile/cache profile。 +- 强制：所有会改变同批执行状态的 intent 必须进入 request-mode 与 step-mode compatibility key；保留 + omitted 与 explicit value 的原始区别，只有 exact key 相同的 request 才能 co-batch。若模型把其他 + request-local 字段纳入 installation key，则这些字段也必须进入 batch key，或由可执行检查保证该模型 + 始终一次只处理一个 request。 +- 禁止：在 batch 内按第一条 request 的 policy 切换 process-wide hooks；把 `None` 正规化成某个显式值； + 只更新 request scheduler 而漏掉 step scheduler。目标 pin 的 `quality` 已进入两套 key；H3 的 refresh + hint 没进入 key，但 H3 `forward` 明确拒绝 `len(prompts) != 1`，不得把该例外推广到 batched adopter。 +- 验收：两套 scheduler 都覆盖 same intent co-batch、accelerated/reference 分离、omitted/explicit 分离； + 每个新 policy input 用 mixed batch 证明 key 完整，或测试 single-request guard。^[PR #5853] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 2c24afda3..ccb1778f0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -27,7 +27,7 @@ confidence: high | HSDP/FSDP、`fully_shard`、DeviceMesh、packed/scalar parameter、FP8 | `checkpoint-distributed`：`DIFF-2b` | `distributed/hsdp.py::{apply_hsdp_to_model,shard_model}` → loader `_load_model_with_hsdp` → `hsdp_fp8.py` | | distributed layerwise offload、AllGather、异构 block、shared buffer | `checkpoint-distributed`：`DIFF-2e` | `offloader/distributed_layerwise_backend.py::{DistributedLayerwiseOffloadHook.initialize_hook,prefetch_layer,DistributedLayerwiseOffloadBackend._allocate_shared_buffers}` | | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | -| modular/multi-DiT、`_dit_modules`、Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2h` | pipeline runtime component list → registry/runner/cache/LoRA/module discovery consumers | +| modular/multi-DiT、`_dit_modules`、request-scoped Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2j` | request batch key → model policy → cache runtime；pipeline component list → shared consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4i` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | @@ -36,7 +36,7 @@ confidence: high |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、async output readiness | `DIFF-1a`–`1e` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT lifecycle | `DIFF-2a`–`2h` | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2j` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4i` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | @@ -199,7 +199,7 @@ confidence: high DP+SP 等其他组合仍缺完整 model×hardware E2E。^[PR #5764] ^[PR #5839] ^[PR #5836] 多 DiT component discovery 与 dotted-path lifecycle 合同见 -[DIFF-2f–2h 专页](rules-component-lifecycle.md)。 +[DIFF-2f–2j 专页](rules-component-lifecycle.md)。 ## 质量阈值与资源辅助 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8e5ffd78d..a27c0e422 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e906f41d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d8c35663` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c9f552627..48a0029be 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e906f41d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d8c35663` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index dc3f165e6..cef9e37f5 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e906f41d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d8c35663` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cdca0718f..06a9f0568 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e906f41d` 验证存在 +- 源码校验：以上路径均已在 `main @ d8c35663` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/bagel/rules.md b/knowledge/repos/vllm-omni/models/bagel/rules.md index 279db1750..340b7f631 100644 --- a/knowledge/repos/vllm-omni/models/bagel/rules.md +++ b/knowledge/repos/vllm-omni/models/bagel/rules.md @@ -46,7 +46,7 @@ refresh/summary 时重新解析该 path，故 enable 后不得替换 `language_m declaration 同时列出两个 alias，也会重复 enable，因为当前 discovery 不按 identity 去重。 **验收**：真实 Bagel component tree 覆盖 Cache-DiT enable→repeated request refresh→summary，断言 -hook/context 始终绑定同一 nested model；目标 cache backend 没有 disable API，future 若增加 worker -shutdown/teardown 必须验证清理同一 target。对 compile、SP、LoRA 与 offload 分别验证同一 object -identity 和 missing-middle 行为。当前 mock 测试只覆盖 Cache-DiT dotted enable/refresh/summary， -不构成真实 cross-consumer 或 teardown 证据。^[PR #5884] +hook/context 始终绑定同一 nested model，disable 清理安装时记录的 target。对 compile、SP、LoRA 与 +offload 分别验证同一 object identity 和 missing-middle 行为。当前 mock 测试覆盖 dotted +enable/refresh/summary；multi-target disable 虽有 mock 覆盖，但没有真实 cross-consumer、异常或 +worker shutdown 证据。^[PR #5884] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0f7443ac1..73380af6b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e906f41d`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d8c35663`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to e906f41d \ + --to d8c35663 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index f99bdb46d..931a2d0fe 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5829", "PR #5836", "PR #5837", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5829", "PR #5836", "PR #5837", "PR #5853", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] confidence: high --- @@ -26,6 +26,8 @@ confidence: high | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | | modular checkpoint、combined/partition task、两套 DiT、shared component | `MMH3-2d` | model index discovery → startup task selection → task-specific transformer/cache lifecycle | +| request `quality`、lossless/high、dynamic Cache-DiT | `MMH3-2e` | request sampling → quality policy → request Cache-DiT runtime → denoise | +| force-refresh hint、once/repeat、reinstall key | `MMH3-2f` | H3 `extra_args` validation → immutable cache config → installation key/refresh context | | DLO、TP-local、resident layers、encoder/VAE staging | `MMH3-3a` | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | `MMH3-3b` | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | `MMH3-3c` | nightly lane → perf JSON → benchmark request encoder/result artifact | @@ -236,6 +238,38 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection 下限，因此不能证明 final merge 的 combined 路径；148.27/158.67 GiB combined 与 86.27 GiB single-load 等数字来自缺硬件/协议/重复次数的 issue comment，不能作为容量保证。^[PR #5720] +## MMH3-2e — quality 映射只决定 request 的 Cache-DiT 目标 + +- 触发：修改公共 `quality` 值、H3 quality policy、startup cache adoption 或 denoise 前 prepare。 +- 强制：公共层只接受 `None`、`lossless`、`high`，并保留 omitted 与 explicit lossless 的区别。 + H3 中 `lossless` 总是无 cache；`high` 总是选择 model-owned conservative profile，即使启动时没有 + cache backend；omitted 仅在启动配置为 Cache-DiT 时恢复 server generic profile，否则无 cache。 + pipeline 必须在参数/任务/step 解析完成后、真实 denoise 紧邻之前 apply plan。 +- 禁止：沿用 PR 早期描述，把无 startup cache 的 `high` 拒绝；让 unsupported quality 静默落入 + lossless；在 request 未显式提供 quality 时由 sync/streaming serving 覆盖 model default。recipe + 把同一组 quality 数字标为 4×H200，而 PR evidence 把它归于 4×L20X SP4；hardware provenance + 未统一前不得引用该表作为任一硬件的可靠 benchmark，更不能外推通用 speedup/quality。 +- 验收：offline、chat、sync video、streaming video 都覆盖 omitted/lossless/high/非法值及显式字段 + 传播；startup cache on/off × 三种 intent 验证 exact installation key，并断言 prepare 先于 diffuse。 + 当前测试广泛覆盖 validation/routing/policy 和 mock 顺序，但真实 Cache-DiT transition 的并发、取消、 + failure rollback 仍缺证据；独立 reviewer 的单卡 B300 观察也显示 `high` 的质量/延迟与四卡表明显 + 不同，只能支持 topology-dependent 边界。共享状态机与 batch 合同见 + [DIFF-2i–2j](../../components/diffusion/rules-component-lifecycle.md)。^[PR #5853] + +## MMH3-2f — force-refresh hint 属于 active profile identity + +- 触发：H3 request `extra_args` 使用 `force_refresh_step_hint` 或 `force_refresh_step_policy`。 +- 强制：hint 是 1-based positive integer 且不超过 `num_inference_steps`，policy 只接受 once/repeat， + 省略 policy 默认 once；没有 active cache target 时两者都拒绝。用 dataclass replace 生成 request-local + config，不修改 startup generic config；installation key 必须包含 hint+policy，因为 Cache-DiT 的 + incremental refresh 把 `None` 解释为保留旧 hint，变更或移除 hint 必须 reinstall hooks。 +- 禁止：接受 bool 作为整数；只给 policy；跨 request 原地修改共享 config；same-key repeated request + 忘记重置 once hint，导致它只在首个 request 生效。 +- 验收：边界 1/steps、0/越界/bool/非法 policy、无 target、hint add/change/remove、once/repeat 和 + repeated same-key request；断言每次 refresh 重建 hint context、generic config 不变，key 变化发生 + disable→enable。当前覆盖为 mock/config 合同，未证明不同 topology 下 hint 对质量/命中率的效果。 + ^[PR #5853] + ## MMH3-3a — H3 DLO 必须保持 loader layout 与 component stage 配对 - 触发：H3 `OffloadPlan`、`dlo_no_use_allgather`、resident layer、text encoder/VAE staging From 9ff6f266d4279459e0071f7d4f49cc3a9bcecfbf Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 18:18:25 +0800 Subject: [PATCH 051/465] knowledge(vllm-omni): sync cfa6d66e (PR #5887) Upstream-Commit: cfa6d66ed6aab7d763142bff0d9aa3489848f650 Upstream-PR: #5887 Knowledge-Delta: scope Qwen-Image dense FlashAttention determinism --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 ++- .../components/diffusion/rules-attention.md | 24 +++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 9 ++++--- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../vllm-omni/models/qwen-image/_index.md | 2 ++ .../vllm-omni/models/qwen-image/rules.md | 22 +++++++++++++++++ 13 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-attention.md create mode 100644 knowledge/repos/vllm-omni/models/qwen-image/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6899d23ad..67ca59239 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e906f41dc8ea12cecdee85a59b62e1e488253d3b + previous_audited_sha: d8c35663b47e54346062e6eac312e7b65bbb6829 audited_ref: main - audited_sha: d8c35663b47e54346062e6eac312e7b65bbb6829 + audited_sha: cfa6d66ed6aab7d763142bff0d9aa3489848f650 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6a34e3518..041dbcdd7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - d8c35663b47e54346062e6eac312e7b65bbb6829`. The canonical machine baseline + cfa6d66ed6aab7d763142bff0d9aa3489848f650`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 041315b80..cf64887c3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d8c35663` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ cfa6d66e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 38c2762f6..1c31cd7c9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d8c35663` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ cfa6d66e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -31,6 +31,7 @@ sources: [] | 理解共享职责和数据流 | [architecture](architecture.md) | | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | +| local FlashAttention deterministic opt-in、dense/non-dense dispatch 与 config propagation | [attention rules](rules-attention.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md new file mode 100644 index 000000000..a35719b16 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -0,0 +1,24 @@ +--- +title: "Diffusion attention 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #5887", vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py] +confidence: high +--- + +# Diffusion attention 规则 + +## DIFF-1f — deterministic flag 只对构造时选定的 local dense FA 生效 + +- 触发：修改 `--fa-deterministic`、diffusion config projection 或 local `FLASH_ATTN` dispatch。 +- 强制：direct CLI 经 engine args/AsyncOmniEngine 传播到 `OmniDiffusionConfig`，attention impl 在构造 + 时捕获；true 只在 dense unmasked `flash_attn_func` 传 `deterministic=True`，false 必须省略 kwarg + 保留 library default。structured deploy 必须由 `_DiffusionConfigProjection` 显式拥有字段。 +- 禁止：把该 flag 描述成全 FlashAttention、跨平台或运行时动态 toggle。piecewise、packed-varlen、 + masked-varlen、dense-varlen fallback 以及 XPU/NPU path 不消费 kwarg，启用时只能 warning once。 +- 验收：direct CLI 与 structured/deploy config 分别做 true/false 端到端传播；dense true/false 精确断言 + kwarg，每条 non-dense path 断言不传且只警告一次，构造后改 config 不得被误认为立即生效。目标 pin + 遗漏 `_DiffusionConfigProjection.fa_deterministic`，所以其派生 config/stage-engine field set 会过滤 + structured path；现有 accuracy fixture 只证明 direct CLI wiring。^[PR #5887] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index ccb1778f0..bbe6f5f01 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -20,7 +20,7 @@ confidence: high | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| -| CUDA Graph、compile、fused solver/norm/RoPE、eager parity、tensor dtype/device、async output | `execution-parity`：`DIFF-1a`–`1e` | `compile.py::regionally_compile` → shared layer/backend → model denoise/solver/output pump | +| CUDA Graph、compile、fused solver/norm/RoPE、FA determinism、eager parity、tensor dtype/device、async output | `execution-parity`：`DIFF-1a`–`1f` | `compile.py::regionally_compile` → shared layer/backend → model denoise/solver/output pump | | seed、request-local generator、guidance=0、并发 RNG、batched generators | `execution-parity`：`DIFF-1b` | `inputs/data.py::OmniDiffusionSamplingParams` → runner `_initialize_generator` → request-batch generator collate | | ModelOpt/checkpoint adapter、weight/scale remap、unknown tensor、resolution path | `checkpoint-distributed`：`DIFF-2a` | `diffusers_loader.py::{_get_checkpoint_adapter,load_weights}` → `modelopt.py::{_resolve_target_and_output_names,adapt}` | | host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | @@ -35,7 +35,7 @@ confidence: high | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | -| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、async output readiness | `DIFF-1a`–`1e` | +| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output readiness | `DIFF-1a`–`1f` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2j` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4i` | @@ -51,7 +51,10 @@ confidence: high - 禁止：只比较 shape、无 NaN 或“能运行”；这些不能证明数值和请求语义等价。 - 验收：固定输入和 request-local generator，对 eager/优化路径逐步比较关键状态并覆盖 零值和最后一步边界。Ming-TTS 的具体反例见 - [Ming-Omni-TTS 规则](../../models/ming-omni-tts/rules.md)。 ^[PR #4341] + [Ming-Omni-TTS 规则](../../models/ming-omni-tts/rules.md)；local FlashAttention deterministic + opt-in 的共享合同见 [DIFF-1f](rules-attention.md)，Qwen accuracy 边界见 + [Qwen-Image 规则](../../models/qwen-image/rules.md)。 + ^[PR #4341] ### DIFF-1b — 随机状态属于请求，零值不是缺省值 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a27c0e422..17c9904a3 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d8c35663` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ cfa6d66e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 48a0029be..3dda9e285 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d8c35663` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ cfa6d66e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cef9e37f5..a1faeb61e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d8c35663` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ cfa6d66e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 06a9f0568..28d7ff51b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d8c35663` 验证存在 +- 源码校验：以上路径均已在 `main @ cfa6d66e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 73380af6b..b2b66ad11 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d8c35663`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ cfa6d66e`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to d8c35663 \ + --to cfa6d66e \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-image/_index.md b/knowledge/repos/vllm-omni/models/qwen-image/_index.md index 971c214f8..93132f896 100644 --- a/knowledge/repos/vllm-omni/models/qwen-image/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-image/_index.md @@ -45,3 +45,5 @@ sources: [vllm_omni/diffusion/models/qwen_image/, vllm_omni/diffusion/registry.p - 审查 qwen_image 任一变体的 CFG-parallel、分布式 VAE 或 DMD2 改动——五变体 共栈,单变体改动先扫其余四个;共享 mixin 见 [Diffusion 组件](../../components/diffusion/_index.md)。 +- 审查 accuracy、regional compile 或 FlashAttention 确定性时见 + [Qwen-Image 规则](rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/qwen-image/rules.md b/knowledge/repos/vllm-omni/models/qwen-image/rules.md new file mode 100644 index 000000000..db695706c --- /dev/null +++ b/knowledge/repos/vllm-omni/models/qwen-image/rules.md @@ -0,0 +1,22 @@ +--- +title: "Qwen-Image 实现规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5887", tests/e2e/accuracy/test_qwen_image.py] +confidence: high +--- + +# Qwen-Image 实现规则 + +## QWENIMG-1a — accuracy gate 必须先证明同 seed repeatability + +- 触发：修改 Qwen-Image accuracy threshold、regional compile 或 fixture 的 deterministic opt-in。 +- 强制：accuracy fixture 显式启用 shared [DIFF-1f](../../components/diffusion/rules-attention.md) 的 + dense FA flag；先以相同 seed 多次比较 Omni↔Omni 唯一性，再用独立 gate 比较 Omni↔Diffusers。 +- 禁止：从单次跨实现 SSIM/PSNR 推断 nondeterminism 已消失；把 accuracy-only opt-in 当成普通 serving + 默认；用跨实现系统偏差解释同实现双稳态，或反向混淆两者。 +- 验收：regional compile + dense FA 下固定完整请求至少重复运行多次，先断言同实现输出稳定，再执行 + SSIM≥0.97、PSNR≥30 的跨实现 gate。PR 的 L20X run 为 0.960/28.188，未通过恢复后的两个 gate； + 合并后评论仍报告 0.958，因此目标证据不能证明 flake 已消除。^[PR #5887] From f4f6827a86982df23a649f05a1e683d8b936d739 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 18:21:57 +0800 Subject: [PATCH 052/465] knowledge(vllm-omni): sync 18bf7e0f (PR #5897) Upstream-Commit: 18bf7e0ff9310000c8c3b5d38e55727cbd94b2ee Upstream-PR: #5897 Knowledge-Delta: close FlashAttention deterministic config projection gap --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules-attention.md | 8 +++++--- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 10 files changed, 16 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 67ca59239..c999cc317 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d8c35663b47e54346062e6eac312e7b65bbb6829 + previous_audited_sha: cfa6d66ed6aab7d763142bff0d9aa3489848f650 audited_ref: main - audited_sha: cfa6d66ed6aab7d763142bff0d9aa3489848f650 + audited_sha: 18bf7e0ff9310000c8c3b5d38e55727cbd94b2ee # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 041dbcdd7..58fada019 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - cfa6d66ed6aab7d763142bff0d9aa3489848f650`. The canonical machine baseline + 18bf7e0ff9310000c8c3b5d38e55727cbd94b2ee`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index cf64887c3..7f8767f28 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ cfa6d66e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 18bf7e0f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1c31cd7c9..d8f68b639 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ cfa6d66e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 18bf7e0f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index a35719b16..5f40e3bf3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py] +sources: ["PR #5887", "PR #5897", vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/config/test_omni_config.py] confidence: high --- @@ -20,5 +20,7 @@ confidence: high masked-varlen、dense-varlen fallback 以及 XPU/NPU path 不消费 kwarg，启用时只能 warning once。 - 验收：direct CLI 与 structured/deploy config 分别做 true/false 端到端传播；dense true/false 精确断言 kwarg，每条 non-dense path 断言不传且只警告一次，构造后改 config 不得被误认为立即生效。目标 pin - 遗漏 `_DiffusionConfigProjection.fa_deterministic`，所以其派生 config/stage-engine field set 会过滤 - structured path；现有 accuracy fixture 只证明 direct CLI wiring。^[PR #5887] + 已将 `fa_deterministic` 加入 `_DiffusionConfigProjection`，因此 `from_kwargs` 不再过滤且派生 + config/stage-engine field set 会分类该字段；PR 引用/运行的既有 classification test 只断言字段分类 + 集合与 projection dataclass 一致，没有以 non-default true 证明 structured/deploy→ + `OmniDiffusionConfig`→backend 的值传播，也没有重跑 Qwen accuracy。^[PR #5887] ^[PR #5897] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 17c9904a3..b2a6ad58f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ cfa6d66e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 18bf7e0f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3dda9e285..6c80ed754 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ cfa6d66e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 18bf7e0f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a1faeb61e..20103b700 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ cfa6d66e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 18bf7e0f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 28d7ff51b..74fa99a6d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ cfa6d66e` 验证存在 +- 源码校验：以上路径均已在 `main @ 18bf7e0f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b2b66ad11..15e9978be 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ cfa6d66e`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 18bf7e0f`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to cfa6d66e \ + --to 18bf7e0f \ --repo \ --mode report-only ``` From 9cf1526c1aa2afd072b4ed644eb277e03611f84d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 18:27:13 +0800 Subject: [PATCH 053/465] knowledge(vllm-omni): sync b15d4e6a (PR #5317) Upstream-Commit: b15d4e6a64f80725a9b33e3fa4bba2fbf100d44c Upstream-PR: #5317 Knowledge-Delta: define batched chat fan-out and aggregation contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 3 +- .../components/serving/rules-batch-chat.md | 50 +++++++++++++++++++ .../vllm-omni/components/serving/rules.md | 2 + knowledge/repos/vllm-omni/models/catalog.md | 4 +- 11 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-batch-chat.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c999cc317..8d4d03c98 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: cfa6d66ed6aab7d763142bff0d9aa3489848f650 + previous_audited_sha: 18bf7e0ff9310000c8c3b5d38e55727cbd94b2ee audited_ref: main - audited_sha: 18bf7e0ff9310000c8c3b5d38e55727cbd94b2ee + audited_sha: b15d4e6a64f80725a9b33e3fa4bba2fbf100d44c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 58fada019..477283513 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 18bf7e0ff9310000c8c3b5d38e55727cbd94b2ee`. The canonical machine baseline + b15d4e6a64f80725a9b33e3fa4bba2fbf100d44c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7f8767f28..79e646d55 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 18bf7e0f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b15d4e6a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d8f68b639..ff973d927 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 18bf7e0f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b15d4e6a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b2a6ad58f..b857ee7d0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 18bf7e0f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b15d4e6a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6c80ed754..a2b38178b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 18bf7e0f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b15d4e6a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 20103b700..19ca3686d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 18bf7e0f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b15d4e6a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 74fa99a6d..7db7a355e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 18bf7e0f` 验证存在 +- 源码校验：以上路径均已在 `main @ b15d4e6a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -29,3 +29,4 @@ sources: [] |---|---| | 理解入口到 engine 的边界 | [architecture](architecture.md) | | 根据 PR 描述直达 request contract、streaming format、artifact readiness、factory lifecycle 或 metrics 的规则组与第一批源码 | [rules 与代码地图](rules.md) | +| batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-batch-chat.md b/knowledge/repos/vllm-omni/components/serving/rules-batch-chat.md new file mode 100644 index 000000000..fee303814 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-batch-chat.md @@ -0,0 +1,50 @@ +--- +title: "Batched Chat Serving 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #5317", vllm_omni/config/endpoint_policy.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/batch_serving.py, tests/e2e/online_serving/test_flux_2_dev_expansion.py, tests/e2e/online_serving/test_qwen3_omni.py, tests/entrypoints/openai/test_batch_chat_completions.py] +confidence: high +--- + +# Batched Chat Serving 规则 + +该 endpoint 是 frontend fan-out/fan-in，不是 scheduler 或 engine native batch；模型 E2E 只证明 +对应输出 surface 可经该 wrapper 返回，不改变模型或 scheduler owner。 + +## SERV-4i — 每个 batch input 必须有唯一、稳定且可取消的 child request + +- 触发：修改 `/v1/chat/completions/batch`、child request 转换、request ID 或并发 task orchestration。 +- 强制：先把每项转换为普通 non-streaming chat request并按 input index 建立稳定 child ID，再并发提交； + fan-in 按原 input 顺序返回。复用 raw Request 时必须隔离 header/request-id 等 mutable state，ID 在并发 + batch 间也须唯一；disconnect、child exception 和 early abort 必须 cancel 并 await 全部未完成 task。 +- 禁止：称其为 engine batching；无上限地按 input 数创建 task；只在单 batch 内 suffix 去重却允许用户 + 重用 base ID 造成跨 batch collision；共享并修改一个 raw Request 而不证明 child isolation。 +- 验收：覆盖有/无 header、相同 user base ID 的并发 batches、顺序不同的 child completion、conversion + error、child exception、disconnect/cancel 和大 batch admission。目标实现只证明单 batch suffix 唯一； + 它复用并修改 raw Request，task 数无界，finally cancel 后不 await，跨 batch collision/泄漏未覆盖。 + ^[PR #5317] + +## SERV-4j — batch response cardinality 永远是一项 input 对一项 choice + +- 触发：普通 chat 新增 output modality/choice shape，或修改 batch choice collapse。 +- 强制：单 content choice 原样保留；恰好一个 content 加一个 audio choice 合成同一 message，顺序无关； + image list 仍是一个 content choice。aggregate choice index 必须等于 input position。 +- 禁止：把 text/audio 两个内部 choice 直接展开成两项 batch choice；把 image content list 的元素数误当 + choice 数；对多个 content、多个 audio 或超过两个 choice 猜测合并。 +- 验收：text、audio+text 两种顺序、image、image+audio 与非法多 choice 精确断言 payload 和 cardinality； + 当前 unit tests 覆盖 collapse shape，Qwen E2E 覆盖 text/audio，FLUX E2E 只检查 choice 数，未验证 image + payload 的 JSON serialization；最终 `model_dump(mode="json")` 也未用 `serialize_as_any`。^[PR #5317] + +## SERV-4k — child 与 aggregation failure 必须按来源分类且 whole-batch 语义明确 + +- 触发：child 返回 `ErrorResponse`、conversion/collapse 失败、usage 聚合或 streaming request。 +- 强制：client conversion/unsupported option 返回结构化 4xx；child `ErrorResponse` 保留其 code；内部 + response-shape/collapse/task failure 返回 5xx。接口若采用 whole-batch failure，就不得混入 partial choices； + usage 合同须明确只汇总 basic prompt/completion/total，或完整合并新增 detail 字段。 +- 禁止：用 `_create_error_response` 的默认 400 包装服务器产生的非法 choice shape；把 valid + `stream=true` 静默改 false 后让客户端误以为请求语义已满足；忽略某个 child usage/error。 +- 验收：malformed child、n>1、first/middle/last child 4xx/5xx、collapse failure、task exception、partial + success、stream request 和 usage details 分别断言 HTTP/status/body。目标实现对 stream 只 server warning， + 任一 child error 返回 whole-batch error，basic usage 求和；collapse 内部失败仍误报默认 400。^[PR #5317] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index f5d4a4c5f..3d4381f05 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -20,6 +20,7 @@ confidence: high | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| | `extra_body`、flattened/nested/canonical/legacy 输入、alias、`negative_prompt`、diffusion request extras | `request-contract`：`SERV-4a`–`4h` | `vllm_omni/entrypoints/openai/diffusion_request_utils.py::{normalize_diffusion_request_args,apply_normalized_diffusion_request_extra_args}` → `serving_chat.py::{OmniOpenAIServingChat._preprocess_chat,OmniOpenAIServingChat.generate_diffusion_images}` | +| batched chat、fan-out/fan-in、sub-request ID、choice collapse、whole-batch error | `batch-chat-contract`：`SERV-4i`–`4k` | `api_server.py::create_batch_chat_completion` → `batch_serving.py::OmniOpenAIServingChatBatch` → ordinary chat completion children | | `chat_template_kwargs`、raw HTTP/SDK `extra_body`、text/audio modalities、choices、空音频 | `chat-multimodal-contract`：`SERV-4c` + 命中模型规则 | upstream `ChatCompletionRequest` → `serving_chat.py::{OmniOpenAIServingChat._preprocess_chat,OmniOpenAIServingChat.chat_completion_full_generator,OmniOpenAIServingChat._create_text_choice,OmniOpenAIServingChat._create_audio_choice}` | | endpoint restriction、route/app-state guard、capability、公开 400 | `endpoint-capability`：`SERV-4c`, `SERV-4d`, `SERV-5d` | endpoint policy → `api_server.py::build_app` assembled app → public handler | | sleep/wake、partial stage/tag、idempotency、ACK、generation admission | `engine-lifecycle`：`SERV-5a`, `SERV-5b` | `entrypoints/async_omni.py::{AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni.generate}` → `worker/base.py::{handle_sleep_task,handle_wake_task}` / `diffusion/worker/diffusion_worker.py` | @@ -41,6 +42,7 @@ confidence: high | `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵 | `SERV-5a`, `SERV-5b`, `SERV-5c` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6c` | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`, `SERV-4b`, `SERV-4c`, `SERV-4d`, `SERV-4e`, `SERV-4f`, `SERV-4g`, `SERV-4h` | +| `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`, `SERV-4j`, `SERV-4k` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | ## SERV-1a — 所有可预判错误在第一个 streaming chunk 前返回 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 15e9978be..8d108bbf8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 18bf7e0f`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b15d4e6a`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 18bf7e0f \ + --to b15d4e6a \ --repo \ --mode report-only ``` From 909ab9c50ca90ad5a7c0fea597000b322c1c955a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 18:36:56 +0800 Subject: [PATCH 054/465] knowledge(vllm-omni): sync 44e38d7b (PR #5864) Upstream-Commit: 44e38d7b3879107e501acfd833765ef07c60ace6 Upstream-PR: #5864 Knowledge-Delta: harden DLO DP waves and multiprocess output lifecycle --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-output-lifecycle.md | 70 +++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 28 +++----- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +- 11 files changed, 91 insertions(+), 30 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8d4d03c98..6c82da90c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 18bf7e0ff9310000c8c3b5d38e55727cbd94b2ee + previous_audited_sha: b15d4e6a64f80725a9b33e3fa4bba2fbf100d44c audited_ref: main - audited_sha: b15d4e6a64f80725a9b33e3fa4bba2fbf100d44c + audited_sha: 44e38d7b3879107e501acfd833765ef07c60ace6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 477283513..3ee1c1606 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - b15d4e6a64f80725a9b33e3fa4bba2fbf100d44c`. The canonical machine baseline + 44e38d7b3879107e501acfd833765ef07c60ace6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 79e646d55..2bcb01b89 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b15d4e6a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 44e38d7b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ff973d927..c8a647330 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b15d4e6a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 44e38d7b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -32,6 +32,7 @@ sources: [] | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | local FlashAttention deterministic opt-in、dense/non-dense dispatch 与 config propagation | [attention rules](rules-attention.md) | +| DLO DP collective wave、per-worker result queue、async pump、SHM ownership 与 shutdown | [multiprocess runtime rules](rules-output-lifecycle.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md new file mode 100644 index 000000000..7705a7c59 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -0,0 +1,70 @@ +--- +title: "Diffusion multiprocess runtime 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #5864", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py] +confidence: high +--- + +# Diffusion multiprocess runtime 规则 + +## DIFF-1d — async output 的 compute 与 output-ready 必须分成两个可观察阶段 + +- 触发：把 diffusion D2H/SHM packing 移到 worker 后台线程，或修改 result pump、`collective_rpc`、 + batch output split。 +- 强制：GPU compute 完成只释放下一次 forward；最终输出必须等 side-stream event、D2H 和 SHM packing + 完成后再 resolve。每个 worker-owned result queue 由唯一 pump reader 分发，batch 结果按 request 映射 + 拆分；step-mode 保留同步路径。 +- 禁止：在 compute-done 时把未完成 device tensor 当最终 artifact；让多个线程同读一个 result queue； + 用同步 `.cpu()` 掩盖 stream ordering 或让下一 step 重写源 buffer。 +- 验收：覆盖 compute-done、output-ready、RPC error、batch split、queue cleanup 与非 CUDA/step-mode + fallback。^[PR #5864] + +## DIFF-1g — 每个 worker result queue 必须有唯一 reader 与显式 SHM owner + +- 触发：request-mode worker result queue、async pump、multi-rank reply 或 tensor/NumPy SHM serialization。 +- 强制：每个 worker 创建独立 result queue，executor 为每个 queue 建唯一 pump；RPC/output future 按 + envelope ID 路由，DP reply 保留 rank tag。递归 pack/unpack 必须覆盖 tagged/RPC envelope、nested + dict/list/tuple、tensor 与大于 1MB 的 non-object ndarray；reader copy 后 close+unlink segment。 +- 禁止：多个 worker 共写一个 queue 后假设 reply 不冲突；pump 与同步 caller 同读；object ndarray + 进入 raw SHM；只释放顶层 handle 而泄漏 nested segment。 +- 验收：真实 spawn workers 各建 queue，传 nested tensor/video/audio ndarray，验证 tag/order/dtype/value， + 所有 pump 停止、queue 关闭、SHM name 不再可 attach。当前测试覆盖成功 round-trip；partial pack、 + unpack/copy 异常时已创建 segment 的回收仍缺 fault-injection。^[PR #5864] + +## DIFF-1h — request-mode shutdown 必须幂等、有界且按 engine→task/IPC 顺序完成 + +- 触发：worker async-output thread、executor process cleanup、inline client 或 orchestrator pre-mark shutdown。 +- 强制：worker 用 sentinel 停 async thread，并证明 thread 已停止后才 teardown 它仍可能访问的 + model/queue/SHM；executor 以全局 deadline join 全部 workers，统一 terminate survivors，再 join + pumps/fail futures。inline client 区分 shutting-down 与 + shutdown-complete，在 lock 内先 close engine 再等待 thread-pool；调用方复用 sampling params 时在 task + 启动前同步 clone。 +- 禁止：把 orchestrator 的 pre-mark 当作 teardown 已完成；每 worker 单独耗完整 timeout；先等待可能 + 阻塞在 engine RPC 的 executor 再 close engine；二次 shutdown 重复释放。 +- 验收：normal、worker shutdown exception、premarked、concurrent/repeated shutdown、blocked RPC、async + output active 与 forced termination 都验证无 live thread/process/queue/SHM，future 到达终态。目标常量 + 为 worker 15s 全局 join、5s terminate grace、async thread 10s、pump 2s；async thread 超时仍存活时 + 虽保留 result queue，却仍先调用 `worker.shutdown()`，model teardown 可能与 live output thread 竞态， + 且没有后续强制 thread/SHM/queue 回收证明。final 2×B300 rerun提供一次普通 TP2 与 + DLO DP2 clean exit，但不替代 fault matrix。^[PR #5864] + +## DIFF-2k — DLO AllGather DP wave 必须完整兼容或不可复用地 fail closed + +- 触发：DLO+AllGather、DP>1 允许 concurrent request，或修改 multi-rank `execute_model` RPC。 +- 强制：只在 DLO+AllGather+DP>1 放宽 single-request pipeline admission；preflight 复用完整 + `RequestBatchSamplingParamsKey`，另比较 canonical `extra_args` 并拒绝空 prompt。shape、CFG、denoise + schedule、output count、LoRA 与任何 future control-flow 字段必须相同；prompt/token length 和 + seed/generator 可不同。multi-rank reply 按 `dp_rank` tag 排序，step mode 从 + `dp_rank * (TP*SP*PP*CFG)` 选择 replica primary result queue。 +- 禁止：为凑 wave 静默改变用户 wait 配置；用 step count+extra_args 代替完整 key；partial collective + timeout 后复用 process group。少于 DP 个 request 时 rank 以 `dp_rank % len(reqs)` 重复执行已有 request， + 只消费前 request-count 个排序输出；这是 correctness fallback 与额外计算，不得称为满 DP throughput。 +- 验收：完整/partial wave、mismatch shape/CFG/steps/output/LoRA/extra_args、空 prompt、不同 seed/token + length、rank tag 重排与 primary-rank topology 都覆盖；timeout 必须将 engine 标记失败、同步 shutdown + 并通知 failure callback。timeout 默认 600s 且在 import 时从 env 读取，不是 request deadline。final + 2×B300 evidence 覆盖 ordinary TP2 与 H3 DLO DP2 的错误 wave→同 engine recovery→无 active child； + 仍只绑定 eager+CUDNN exact topology，不能推广到一般 DP/SP 或替代 per-replica SP/mmap redesign。 + ^[PR #5864] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index bbe6f5f01..9bac797ab 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -20,14 +20,15 @@ confidence: high | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| -| CUDA Graph、compile、fused solver/norm/RoPE、FA determinism、eager parity、tensor dtype/device、async output | `execution-parity`：`DIFF-1a`–`1f` | `compile.py::regionally_compile` → shared layer/backend → model denoise/solver/output pump | +| CUDA Graph、compile、fused solver/norm/RoPE、FA determinism、eager parity、tensor dtype/device、async output/teardown | `execution-parity`：`DIFF-1a`–`1h` | `compile.py::regionally_compile` → shared layer/backend → model denoise/output pump/shutdown | | seed、request-local generator、guidance=0、并发 RNG、batched generators | `execution-parity`：`DIFF-1b` | `inputs/data.py::OmniDiffusionSamplingParams` → runner `_initialize_generator` → request-batch generator collate | | ModelOpt/checkpoint adapter、weight/scale remap、unknown tensor、resolution path | `checkpoint-distributed`：`DIFF-2a` | `diffusers_loader.py::{_get_checkpoint_adapter,load_weights}` → `modelopt.py::{_resolve_target_and_output_names,adapt}` | | host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | | HSDP/FSDP、`fully_shard`、DeviceMesh、packed/scalar parameter、FP8 | `checkpoint-distributed`：`DIFF-2b` | `distributed/hsdp.py::{apply_hsdp_to_model,shard_model}` → loader `_load_model_with_hsdp` → `hsdp_fp8.py` | | distributed layerwise offload、AllGather、异构 block、shared buffer | `checkpoint-distributed`：`DIFF-2e` | `offloader/distributed_layerwise_backend.py::{DistributedLayerwiseOffloadHook.initialize_hook,prefetch_layer,DistributedLayerwiseOffloadBackend._allocate_shared_buffers}` | +| DLO+AllGather、DP concurrent collective wave、per-worker result queue、shutdown | `checkpoint-distributed`/`execution-parity`：`DIFF-2k`, `DIFF-1d/1g/1h` | request compatibility key → multi-rank RPC → worker-owned queue/pump → bounded teardown | | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | -| modular/multi-DiT、`_dit_modules`、request-scoped Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2j` | request batch key → model policy → cache runtime；pipeline component list → shared consumers | +| modular/multi-DiT、`_dit_modules`、request-scoped Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2j` | request batch key → model policy/cache runtime；pipeline component list → shared consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4i` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | @@ -35,8 +36,8 @@ confidence: high | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | -| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output readiness | `DIFF-1a`–`1f` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2j` | +| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1h` | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2k` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4i` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | @@ -74,19 +75,8 @@ confidence: high - 禁止：依赖 process 默认 fp32/CPU 或隐式 cast/move；只测 shape 而不执行 BF16 consumer。 - 验收：BF16、错长、batch reorder 和 graph/eager 用例断言 consumer 收到相同 device/dtype、 alias 与结果。 ^[PR #5067] ^[PR #5068] ^[PR #5174] ^[PR #5981] -### DIFF-1d — async output 的 compute 与 output-ready 必须分成两个可观察阶段 - -- 触发：把 diffusion D2H/SHM packing 移到 worker 后台线程，或修改 - `ResultPumpThread`、`collective_rpc`、batch output split。 -- 强制：GPU compute 完成只释放下一次 forward；最终输出必须等 side-stream event、D2H - 和 SHM packing 完成后再 resolve。`result_mq` 由唯一 pump reader 分发，batch 结果按 - request 映射拆分；step-mode 保留同步路径。 -- 禁止：在 compute-done 时把未完成的 device tensor 当成最终 artifact；让多个线程同时 - 消费 result queue；用同步 `.cpu()` 掩盖 stream ordering 或让下一 step 重写源 buffer。 -- 验收：分别覆盖 compute-done、output-ready、RPC error、batch split 和 queue cleanup； - 在非 CUDA/step-mode 下证明同步 fallback 仍可构造。首批测试看 - `tests/diffusion/test_async_output_worker.py`、`test_result_pump.py` 和 - `tests/diffusion/test_ipc_async.py`。 +async output readiness、per-worker result channel 与 shutdown 合同见 +[DIFF-1d/1g/1h 专页](rules-output-lifecycle.md)。 ### DIFF-1e — fused layer 必须保留精度、布局与平台分派合同 @@ -193,8 +183,8 @@ confidence: high 不同 block size 验证 small/large hook 都满足 collective size equality、重建权重和 repoint。 当前回归用 mocked collective 检查 8/32-element、dp=2 合同；没有真实 multi-rank collective 覆盖，PR 的 8×NPU H3 E2E 说明也缺少已填写的 commit 证据，不能升级为验证结论。 ^[PR #5802] - DP concurrent wave 只可在 DLO+AllGather 开启，并从 `sampling_params.extra_args` 比较 canonical - signature；no-AllGather 的 TP-local request 不得误走 fused batch。文档 compatibility matrix 中 + DLO DP collective-wave admission 见 [DIFF-2k](rules-output-lifecycle.md)；no-AllGather 的 + TP-local request 不得误走 fused batch。文档 compatibility matrix 中 “accepted” 默认只代表配置/源码 guard；PR #5836 的 predecessor partition-path commit 为 H3 USP4+HSDP4+no-AllGather+8 resident layers 提供一次 4×H100、3/3 completion/perf run，故该 exact 组合不再是纯 config evidence，但没有 ordinary-offload 数值/输出质量对照，且 profiler 开启、 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b857ee7d0..7ecdbb7aa 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b15d4e6a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 44e38d7b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a2b38178b..61b2fc58a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b15d4e6a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 44e38d7b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 19ca3686d..55fbe0830 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b15d4e6a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 44e38d7b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7db7a355e..ad883d9ce 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b15d4e6a` 验证存在 +- 源码校验：以上路径均已在 `main @ 44e38d7b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8d108bbf8..e28c1384c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b15d4e6a`（2026-08-07）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 44e38d7b`（2026-08-08）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to b15d4e6a \ + --to 44e38d7b \ --repo \ --mode report-only ``` From 9a739b1e65be8d4728cb710c846d948249d15e71 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 18:45:10 +0800 Subject: [PATCH 055/465] knowledge(vllm-omni): sync fd0e9218 (PR #5139) Upstream-Commit: fd0e92183a9f265cfe43deadca7c01501af91698 Upstream-PR: #5139 Knowledge-Delta: design-module ownership map and archive routing --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 7 ++-- .../components/scheduler/architecture.md | 7 ++-- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/docs/design-doc-map.md | 33 +++++++++++-------- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/qwen-omni/_index.md | 4 +-- .../models/qwen-omni/architecture.md | 6 ++-- 13 files changed, 44 insertions(+), 33 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6c82da90c..ee25757c5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b15d4e6a64f80725a9b33e3fa4bba2fbf100d44c + previous_audited_sha: 44e38d7b3879107e501acfd833765ef07c60ace6 audited_ref: main - audited_sha: 44e38d7b3879107e501acfd833765ef07c60ace6 + audited_sha: fd0e92183a9f265cfe43deadca7c01501af91698 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3ee1c1606..8db4582b2 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 44e38d7b3879107e501acfd833765ef07c60ace6`. The canonical machine baseline + fd0e92183a9f265cfe43deadca7c01501af91698`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2bcb01b89..9439b4c85 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 44e38d7b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fd0e9218` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c8a647330..0fdda6837 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 44e38d7b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ fd0e9218` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7ecdbb7aa..687ed759a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 44e38d7b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fd0e9218` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 61b2fc58a..db59a50e0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 44e38d7b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fd0e9218` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 55fbe0830..64ddafa65 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -4,17 +4,18 @@ created: 2026-07-16 updated: 2026-09-02 type: index tags: [vllm-omni, components, scheduler] -sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, vllm_omni/core/prefix_cache.py, docs/design/module/ar_module.md] +sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, vllm_omni/core/prefix_cache.py, docs/design/module/ar_runtime.md, docs/design/module/archive/ar_module.md] --- # Scheduler（AR/生成请求调度） - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 44e38d7b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fd0e9218` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） -- 官方设计文档：`docs/design/module/ar_module.md`（继承关系、请求流转图） +- 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 + `docs/design/module/archive/ar_module.md` 保留，须对当前代码复核。 - 测试入口：`tests/core/` - 主要职责：AR/生成 stage 的请求调度（继承 vLLM Scheduler）、跨 stage KV transfer 的调度面、 chunk/full-payload 输入等待状态机、omni tensor prefix cache diff --git a/knowledge/repos/vllm-omni/components/scheduler/architecture.md b/knowledge/repos/vllm-omni/components/scheduler/architecture.md index 256c09fcb..2bf080b17 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/architecture.md +++ b/knowledge/repos/vllm-omni/components/scheduler/architecture.md @@ -4,13 +4,14 @@ created: 2026-07-16 updated: 2026-09-02 type: architecture tags: [vllm-omni, components, scheduler] -sources: [docs/design/module/ar_module.md, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/prefix_cache.py, vllm_omni/worker/gpu_ar_model_runner.py] +sources: [docs/design/module/ar_runtime.md, docs/design/module/archive/ar_module.md, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/prefix_cache.py, vllm_omni/worker/gpu_ar_model_runner.py] --- # Scheduler 共享架构 -以下事实在 `main @ 5215e03a` 复核；官方叙述见 -`docs/design/module/ar_module.md`（含继承 classDiagram 与请求流转 flowchart）。 +以下事实在 `main @ 5215e03a` 复核；active draft source map 是 +`docs/design/module/ar_runtime.md`。继承 classDiagram 与请求流转 flowchart 仅存于 +`docs/design/module/archive/ar_module.md`，属于历史叙述，仍须对当前代码复核。 ## 继承链（对 vLLM 的扩展方式） diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ad883d9ce..e3d86c2a5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 44e38d7b` 验证存在 +- 源码校验：以上路径均已在 `main @ fd0e9218` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index 7d29dd34c..74da5da21 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -4,28 +4,35 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, docs] -sources: ["PR #5833", "PR #5839", docs/.nav.yml, docs/design/index.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] +sources: ["PR #5139", "PR #5833", "PR #5839", docs/.nav.yml, docs/design/index.md, docs/design/module/, docs/design/module/archive/README.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] --- # 官方设计文档地图（docs/design/**） -`main @ 62bf8d63` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 按 architecture、runtime/stage、 -communication、diffusion acceleration、infrastructure/performance 和 module 编排。本轮新增 DLO 与 -quantization design，`cpu_offload_diffusion.md` 重命名为 `cpu_offload.md` 并保留 MkDocs redirect， -同时移除 `ray_based_execution.md`，即删除一项误导性的 supported-design surface。目标 pin 虽仍暴露 -`vllm_omni/distributed/ray_utils/`、`worker_backend` 与 `ray_address`，但 diffusion executor 明确 -`NotImplementedError("ray backend is not yet supported.")`，placement-group/actor helpers 也没有 -live caller；这些残留不是 runtime support 证据。索引仍未列 -`docs/design/ar_diffusion_pipeline_capability.md`；找 spec 时仍须检查 `docs/design/` 实际文件树。 +`main @ fd0e9218` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 按 architecture、feature、 +infrastructure/performance 和 21 个 module 页面编排。module frontmatter 的 owner、代码路径、依赖与 +验证路径可作 review 首跳 source map；但本批页面全部是 `status: draft`，其中 invariant 编号也明确 +只是 candidate，不能覆盖当前代码与测试。尤其 `module/vllm_omni_config.md` 是 +`deferred-pending-refactor`，没有稳定 precedence、环境变量捕获、canonical config 或 invariant namespace。 + +旧四页已移入 `module/archive/` 并从导航移除，只能作为历史叙述；有用结论须先与当前代码/测试复核。 +上游刻意没有为旧 URL 留 redirect，因此外部深链会 404。owner 与 `required_reviewers` 是不同角色， +不能用 document steward/technical owner 冒充独立验证批准。 + +此前 #5833/#5839 增加 DLO 与 quantization design；`cpu_offload_diffusion.md` 重命名为 +`cpu_offload.md` 且保留 MkDocs redirect，并删除误导性的 `ray_based_execution.md`。目标 pin 仍有 +`distributed/ray_utils/`、`worker_backend`、`ray_address`，但 diffusion executor 明确报 Ray backend +尚未支持，placement-group/actor helpers 也没有 live caller，不能据残留宣称 runtime support。 +`docs/design/ar_diffusion_pipeline_capability.md` 仍未列入索引，找 spec 时还须检查实际文件树。 | 官方文档 | 一句话 | 知识树 owner | |---|---|---| | `architecture_overview.md` | 三类模型拓扑（DiT 主/AR 主/AR+DiT）、OmniRouter/EntryPoints/AR/Diffusion/OmniConnector、E/P/D/G 解耦、CFG companion 流 | [serving](../components/serving/architecture.md) + [distributed](../components/distributed/architecture.md) | -| `module/async_omni_architecture.md` | 五层运行时（API→Engine→Orchestration→Communication→Execution），ZMQ + janus 队列，Qwen3-Omni worked example | [serving](../components/serving/architecture.md)、[qwen-omni](../models/qwen-omni/architecture.md) | -| `module/ar_module.md` | AR 模块继承链与请求流转 | [scheduler](../components/scheduler/architecture.md) + [model-executor](../components/model-executor/architecture.md) | +| `module/{entrypoints,vllm_omni_config,input_output_modality_contracts,error_contracts,engine_orchestration,stage_runtime,omni_connector,model_integration,ar_runtime}.md` | serving/config/I-O/error/engine/stage/connector/model/AR 的 draft ownership 与 source map；candidate invariant 尚非规范 | 对应 [serving](../components/serving/_index.md)、[configuration](../components/configuration/_index.md)、[distributed](../components/distributed/_index.md)、[scheduler](../components/scheduler/_index.md)、[model-executor](../components/model-executor/_index.md) | | `feature/omni_async_output_materialization.md` | AR decode 关键路径外的异步 D2H 与 Omni payload 构造；含 live connector ownership、平台验证范围和同步 fallback | [model-executor](../components/model-executor/_index.md) | -| `module/dit_module.md` | Diffusion 引擎/调度/worker/pipeline/加速组件 | [diffusion](../components/diffusion/architecture.md) | -| `module/entrypoint_module.md` | **stub（"update soon"）——上游文档缺口** | [serving](../components/serving/_index.md) | +| `module/diffusion/{index,diffusion_runtime,diffusion_model_integration,continuous_batching,parallelism,offloader}.md` | diffusion draft module 边界与首跳路径 | [diffusion](../components/diffusion/architecture.md) | +| `module/{execution_platforms,cache_management,quantization,observability,profiling,benchmarking}.md` | 横切模块的 draft owner/source map | 对应组件 owner；行为仍以代码和测试为准 | +| `module/archive/{async_omni_architecture,ar_module,dit_module,entrypoint_module}.md` | 历史快照，不是 active contract；旧路径无 redirect | 仅作 provenance，改动路由到 active module 页 | | `feature/disaggregated_inference.md` + `omni_connectors/*` | connector 选择矩阵与逐后端 spec | [distributed](../components/distributed/connector-backends.md) | | `feature/diffusion_continuous_batching.md` + `user_guide/diffusion/execution_modes.md` | 合并后的开发者设计与用户配置入口：request/step 执行、两种 batching、统一 output stream | [diffusion](../components/diffusion/step-and-batching.md) | | `feature/async_chunk.md` | 跨 stage 分块流式 | [distributed](../components/distributed/async-chunk.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e28c1384c..c34598d5c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 44e38d7b`（2026-08-08）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fd0e9218`（2026-08-08）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 44e38d7b \ + --to fd0e9218 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index af930f26b..4f32459fa 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -19,8 +19,8 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline - deploy YAML：`qwen2_5_omni.yaml`（1×H100 验证）、`qwen3_omni_moe.yaml` （2×H100 验证）、`qwen3_omni_moe_mori_intranode.yaml`、`qwen3_tts.yaml` （+ `_forced_aligner`、`_high_concurrency` 变体） -- 官方文档：`docs/design/module/async_omni_architecture.md`（以 Qwen3-Omni 为 - worked example 的分层运行时 spec）、 +- 官方历史文档：`docs/design/module/archive/async_omni_architecture.md`（以 Qwen3-Omni 为 + worked example 的分层运行时快照，非 active spec）、 `docs/design/qwen3_omni_tts_performance_optimization.md`（性能优化实录） ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md index 6b380d728..790288ecb 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-07-16 type: architecture tags: [vllm-omni, models, qwen-omni] -sources: [docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/async_omni_architecture.md, vllm_omni/model_executor/models/registry.py] +sources: [docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/model_executor/models/registry.py] --- # Qwen-Omni 家族拓扑与性能结论 @@ -37,8 +37,10 @@ sources: [docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/mo ## 运行时参照 -家族是 `docs/design/module/async_omni_architecture.md` 分层运行时（API → +家族曾以 `docs/design/module/archive/async_omni_architecture.md` 的分层运行时（API → Engine → Orchestration → Communication → Execution）的官方 worked example； +该页现为历史快照，active draft source map 是 `module/engine_orchestration.md` 与 +`module/stage_runtime.md`，所以 worked example 必须对当前源码复核； 相关共享机制：talker 的 prefix-cache 关键 key 合同见 [Scheduler 规则 SCHED-1a](../../components/scheduler/rules.md)，`talker_mtp` 路由与 runner 预处理合同见 From 68802995c110bb2e6b84e97b0ae1d73ecc73fa30 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 18:57:52 +0800 Subject: [PATCH 056/465] knowledge(vllm-omni): sync 81b48e83 (PR #5824) Upstream-Commit: 81b48e83ecb228712890d48658b212c90441d7d7 Upstream-PR: #5824 Knowledge-Delta: fail-closed MiniMax H3 fused-source loading --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/architecture.md | 2 +- .../components/model-executor/rules.md | 21 ++++---- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +- .../vllm-omni/models/minimax-h3/_index.md | 10 ++-- .../models/minimax-h3/rules-loading.md | 51 +++++++++++++++++++ .../vllm-omni/models/minimax-h3/rules.md | 30 ++--------- 14 files changed, 86 insertions(+), 50 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ee25757c5..a100cc396 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 44e38d7b3879107e501acfd833765ef07c60ace6 + previous_audited_sha: fd0e92183a9f265cfe43deadca7c01501af91698 audited_ref: main - audited_sha: fd0e92183a9f265cfe43deadca7c01501af91698 + audited_sha: 81b48e83ecb228712890d48658b212c90441d7d7 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8db4582b2..1998142f4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - fd0e92183a9f265cfe43deadca7c01501af91698`. The canonical machine baseline + 81b48e83ecb228712890d48658b212c90441d7d7`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9439b4c85..6e79d3a5b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fd0e9218` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 81b48e83` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0fdda6837..15be9afe0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fd0e9218` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 81b48e83` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 687ed759a..1a86d396c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fd0e9218` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 81b48e83` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index db59a50e0..a8d2bcf09 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fd0e9218` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 81b48e83` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/architecture.md b/knowledge/repos/vllm-omni/components/model-executor/architecture.md index b85513ab8..588d1261b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/architecture.md +++ b/knowledge/repos/vllm-omni/components/model-executor/architecture.md @@ -64,7 +64,7 @@ KV-only sender 可只保留 KV manager，不另外创建无 consumer 的 payload 和 `gate_proj`/`up_proj`，HF 同名参数由普通 loader 直接写入，TP plan 也指向这些 分离名称。只有 310P 平台 overlay 在 `prepare_qkv_weights()` 内为本地执行临时拼接 QKV weight/bias；这不表示共享模型或 checkpoint loader 已改为 fused parameter。未来再引入 -共享 fusion 时的验收门禁见 [EXEC-2b](rules.md#exec-2b-未来引入-fused-shard-时必须按布局数值闭环)。 +共享 fusion 时的验收门禁见 [EXEC-2b](rules.md#exec-2b-fused-shard-必须按-source-完整性与布局数值闭环)。 ## 怎样判断问题归属 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 64a962cf0..41504b4bf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5074", "PR #5310", "PR #5610", "PR #5777", "PR #5792", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5074", "PR #5310", "PR #5610", "PR #5777", "PR #5792", "PR #5824", "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -21,7 +21,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | stage TP/PP/DP、devices、replica、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | -| 准备引入 fused projection、HF shard 拼装、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | 先确认目标 main 是否已有 fused parameter；当前 Qwen3 code predictor 为分离 projection | +| fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | H3 text encoder fused owner；其他模型先确认目标 main 是否已有 fused parameter | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | 审查组 | 什么时候触发 | 规则 ID | @@ -169,17 +169,18 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 记录峰值显存和 dtype。Krea 2 的具体约束见 [Krea 2 规则](../../models/krea2/rules.md)。 ^[PR #4730] -### EXEC-2b — 未来引入 fused shard 时必须按布局数值闭环 +### EXEC-2b — fused shard 必须按 source 完整性与布局数值闭环 -> 当前 `main @ 78c144f3` 的 Qwen3 code predictor 使用分离 q/k/v 和 gate/up projection； -> PR #4958 的 fused 实现已因 CI failure 被 PR #5777 整体回退。本规则是对未来再引入 -> 类似优化的门禁，不描述当前 runtime 机制。 +> H3 text encoder 当前有 fused QKV 与 gate/up owner；Qwen3 code predictor 仍使用分离 +> projection，其 PR #4958 fusion 已由 PR #5777 回退，不能把 H3 现状外推给 Qwen。 - 触发：准备合并 q/k/v、gate/up 等 projection，修改 HF shard 映射、wrapper/talker loader、packed projection TP plan 或平台 override。 - 强制：weight 与可选 bias 都按 forward split 的同一顺序数值拼装；部分 shard 和 - 整组 shard 缺失都硬失败。所有声明同一 fused module 的 consumer 必须委托给共享 - loader，wrapper 并恢复 returned loaded-name 的模型前缀；平台 override 同步使用新 + 整组 shard 缺失都硬失败。bookkeeping 必须按 source shard id；expected 集合从实际 fused + owner 预置，才能捕获零 shard，不能以任一 source 命中的 target name 代表完整。所有声明 + 同一 fused module 的 consumer 必须委托给共享 loader，wrapper 并恢复 returned loaded-name + 的模型前缀；平台 override 同步使用新 fused 属性与共享 split helper。 - 禁止：不得用逐 tensor `default_weight_loader` 绕过 fused assembler；不得只记录 skipped shard 后让随机初始参数继续；不得只用 shape 或 returned-name 断言顺序正确。GQA @@ -187,4 +188,6 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 未有 TP-aware packing/loading/split 与 TP=2 测试时，TP plan 保持空。 - 验收：数值比较 fused 参数与 `cat([q,k,v])`/`cat([gate,up])`，覆盖 bias、 部分/整组缺 shard、每个 consumer 的委托和前缀；以非等 q/KV width 证明 split 能识别 - GQA 错序。平台测试或静态 guard 证明不再引用已删除的 projection 属性。 ^[PR #4958] ^[PR #5777] + GQA 错序。平台测试或静态 guard 证明不再引用已删除的 projection 属性。H3 eager text + encoder 还须对任何未加载 plain retained parameter 在启动时硬失败；unknown checkpoint key + 可继续告警，因为它不会留下 model parameter 未初始化。^[PR #4958] ^[PR #5777] ^[PR #5824] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 64ddafa65..f28e3b07c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fd0e9218` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 81b48e83` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e3d86c2a5..9c8ea1c20 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fd0e9218` 验证存在 +- 源码校验：以上路径均已在 `main @ 81b48e83` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c34598d5c..1a509881b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fd0e9218`（2026-08-08）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 81b48e83`（2026-08-08）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to fd0e9218 \ + --to 81b48e83 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 11d991dff..8fe6ccafe 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -41,6 +41,9 @@ confidence: high 匹配，不带 `transformer.` owner prefix。checkpoint 的 grouped-QKV reorder 和 fused-MLP gate/up split 在 `MiniMaxH3DiTModel.load_weights()` 内先执行，再交给当前 vLLM loader， 以保留 TP shard 与 FP8 online-processing wrapper。 +- H3 text encoder 的 eager loader 按 source shard 分别证明 fused q/k/v 与 gate/up 完整，且任何 + 未加载 plain retained parameter 都令启动失败；pipeline 批量上报 encoder parameter 的严格性 + 依赖这项保证。video/audio VAE 没有等价 completeness guarantee，不能由此推断。见 MMH3-1f。 - H3 online FP8 当前不得与 layerwise offload 组合：offload 产生的 weight stride 会被 Cutlass FP8 kernel 拒绝。变更量化覆盖范围时，除逐层命中/排除与加载转换测试外， 必须同时保护 joint video/audio 质量；peak memory 只作为同 case report，不是稳定上界。 @@ -84,5 +87,6 @@ PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、t ## 审查入口 -H3 input matrix/media ingress 以及 online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 -[MiniMax H3 rules](rules.md#direct-代码快速入口)。 +H3 input matrix/media ingress，以及 text-encoder completeness、online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 +[MiniMax H3 rules](rules.md#direct-代码快速入口)；checkpoint transform、quantized loader 与 +text-encoder fused-source 完整性正文见 [loading rules](rules-loading.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md new file mode 100644 index 000000000..69d4a2dde --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md @@ -0,0 +1,51 @@ +--- +title: "MiniMax H3 加载规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py] +confidence: high +--- + +# MiniMax H3 加载规则 + +## MMH3-1a — component namespace 与 checkpoint transform 必须在 active loader 前闭合 + +- 触发：修改 H3 online FP8 覆盖范围、`ignored_layers`、linear prefix、QKV/MLP checkpoint + 变换或 TP loader。 +- 强制：pipeline 先把 structured quantization config 解析到 `transformer` component，再构造 + DiT；因此内部 prefix 是 `blocks.*`、`token_refiner.*` 等 component-relative 名称，不带 + `transformer.`。每个 eligible linear 显式传 prefix/quant config，patch、timestep、final + projection 显式保持非量化。grouped-QKV reorder 与 fused `fc1` gate/up split 在 + `MiniMaxH3DiTModel.load_weights()` 先完成，再调用当前 parameter 的 active weight loader， + 让 TP shard 与 FP8 `online_process_loader` 看到最终布局。 +- 禁止：用 parent prefix 代替 exact leaf `ignored_layers`；一边已 resolve component、一边仍 + 要求 `transformer.`；在 linear 构造后包掉 active loader，或把 reorder/split 放到 FP8 + wrapper 之后。 +- 验收：枚举默认 quantized/full-precision prefix 集合，逐个验证 exact ignored leaf；QKV + 与 gate/up sentinel 断言传给 active loader 的顺序和 shard id，至少覆盖 TP 与 online FP8。 + ^[PR #5737] + +NPU INT8 online 同样复用 component-relative prefix 与 active loader：BF16 checkpoint 在逐层 +load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection 保持未量化； +`ignored_layers` 必须能用 fused owner 名（如 `blocks.0.attn.qkv_proj`）精确跳过。NPU +`QuantBatchMatmulV3` 的 65535 output 限制只按 per-partition width 判断，超限 layer 回退 +`UnquantizedLinearMethod`，更高 TP 可能让同一层重新满足 INT8。^[PR #5706] + +## MMH3-1f — text encoder eager load 必须证明每个 source shard 完整 + +- 触发：修改 H3 Qwen3-VL text encoder 的 checkpoint 映射、QKV/gate-up fused loader、保留层 + 集合，或 pipeline 对 eager component 的 loaded-name 上报。 +- 强制：按实际 fused module 预置 expected source 集合，并按 shard id 分别记录 q/k/v 与 + gate/up；缺单个 source、整组零 source 或任何 plain retained parameter 都在启动时 + `RuntimeError`。pipeline 只有在 encoder 自身完成此验证后，才可把其全部 parameter 上报给 + runner strict check；额外 unknown checkpoint key 可告警跳过，因为不会留下 model parameter + 未初始化。 +- 禁止：任一 source 命中后用 fused target name 宣称完整；只 warning 后保留 `torch.empty` + 行继续执行；把 encoder 的完整性保证外推给同样 eager 构造、但尚无等价验证的 video/audio VAE。 +- 验收：distinct sentinel 数值逐段核对 q/k/v（含 GQA 非等宽）与 gate/up 行布局；分别覆盖每个 + source 缺失、整组缺失和 plain parameter 缺失。真实 checkpoint 的 index/name coverage 可证明 + 映射覆盖，但未执行真实完整加载时，不能声称 shape/dtype/full-load 已回归；本改动本身 + 未改变 shape/dtype handling。共享 fused-loader 门禁见 + [EXEC-2b](../../components/model-executor/rules.md#exec-2b-fused-shard-必须按-source-完整性与布局数值闭环)。^[PR #5824] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 931a2d0fe..86cad6e73 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5829", "PR #5836", "PR #5837", "PR #5853", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5853", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] confidence: high --- @@ -16,12 +16,13 @@ confidence: high | PR 描述信号 | 规则 | 第一批源码 | |---|---|---| -| online FP8、`ignored_layers`、component prefix | `MMH3-1a` | `pipeline_minimax_h3.py::_resolve_component_quant_config` → `MiniMaxH3DiTModel` linear prefix | -| grouped QKV、fused MLP、weight loader、TP | `MMH3-1a` | `minimax_h3_transformer.py::MiniMaxH3DiTModel.load_weights` → active vLLM loader | +| online FP8、`ignored_layers`、component prefix | [MMH3-1a](rules-loading.md#mmh3-1a-component-namespace-与-checkpoint-transform-必须在-active-loader-前闭合) | `pipeline_minimax_h3.py::_resolve_component_quant_config` → `MiniMaxH3DiTModel` linear prefix | +| grouped QKV、fused MLP、weight loader、TP | [MMH3-1a](rules-loading.md#mmh3-1a-component-namespace-与-checkpoint-transform-必须在-active-loader-前闭合) | `minimax_h3_transformer.py::MiniMaxH3DiTModel.load_weights` → active vLLM loader | | FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | | RMSNorm、RoPE、96/128 rotary dim、fused backend | `MMH3-1c` | `MiniMaxH3Attention` → shared `RMSNorm`/`RotaryEmbedding` → platform dispatch | | RainFusion、block sparse、video layout、NPU INT8 | `MMH3-1d` | packed sequence → attention metadata/backend plan；quant config → prefixed linear → loader/post-load | | TRTLLM、ragged packed metadata、SAGE、Skip-Softmax、Blackwell default | `MMH3-1e` | H3 metadata/roles → TRTLLM packed trim/quant gate → platform default | +| text encoder、missing q/k/v 或 gate/up、eager load bookkeeping | [MMH3-1f](rules-loading.md#mmh3-1f-text-encoder-eager-load-必须证明每个-source-shard-完整) | `encoder.py::_load_weights` → `pipeline_minimax_h3.py::load_weights` strict report | | FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | @@ -32,29 +33,6 @@ confidence: high | RTX 5090/4090、24/32 GiB、consumer profile | `MMH3-3b` | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | `MMH3-3c` | nightly lane → perf JSON → benchmark request encoder/result artifact | -## MMH3-1a — component namespace 与 checkpoint transform 必须在 active loader 前闭合 - -- 触发：修改 H3 online FP8 覆盖范围、`ignored_layers`、linear prefix、QKV/MLP checkpoint - 变换或 TP loader。 -- 强制：pipeline 先把 structured quantization config 解析到 `transformer` component，再构造 - DiT；因此内部 prefix 是 `blocks.*`、`token_refiner.*` 等 component-relative 名称，不带 - `transformer.`。每个 eligible linear 显式传 prefix/quant config，patch、timestep、final - projection 显式保持非量化。grouped-QKV reorder 与 fused `fc1` gate/up split 在 - `MiniMaxH3DiTModel.load_weights()` 先完成，再调用当前 parameter 的 active weight loader， - 让 TP shard 与 FP8 `online_process_loader` 看到最终布局。 -- 禁止：用 parent prefix 代替 exact leaf `ignored_layers`；一边已 resolve component、一边仍 - 要求 `transformer.`；在 linear 构造后包掉 active loader，或把 reorder/split 放到 FP8 - wrapper 之后。 -- 验收：枚举默认 quantized/full-precision prefix 集合，逐个验证 exact ignored leaf；QKV - 与 gate/up sentinel 断言传给 active loader 的顺序和 shard id，至少覆盖 TP 与 online FP8。 - ^[PR #5737] - -NPU INT8 online 同样复用 component-relative prefix 与 active loader：BF16 checkpoint 在逐层 -load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection 保持未量化； -`ignored_layers` 必须能用 fused owner 名（如 `blocks.0.attn.qkv_proj`）精确跳过。NPU -`QuantBatchMatmulV3` 的 65535 output 限制只按 per-partition width 判断，超限 layer 回退 -`UnquantizedLinearMethod`，更高 TP 可能让同一层重新满足 INT8。^[PR #5706] - ## MMH3-1b — joint video/audio quality 与 offload 兼容性必须一起验收 - 触发：改变 H3 FP8 layer scope、kernel/loader、quality threshold、offload 或 nightly lane。 From bf690a742dfb95dad576ace0c5eb89d48d83539e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 19:02:48 +0800 Subject: [PATCH 057/465] knowledge(vllm-omni): sync f663d120 (PR #5871) Upstream-Commit: f663d120c78e225976d6cd07b2f93a5c5e13012d Upstream-PR: #5871 Knowledge-Delta: frozen review routing and untrusted-head safety --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../guides/review-execution-contract.md | 10 +++++++-- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- knowledge/repos/vllm-omni/review/_index.md | 22 +++++++++++++++---- 11 files changed, 37 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a100cc396..812eeaeed 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fd0e92183a9f265cfe43deadca7c01501af91698 + previous_audited_sha: 81b48e83ecb228712890d48658b212c90441d7d7 audited_ref: main - audited_sha: 81b48e83ecb228712890d48658b212c90441d7d7 + audited_sha: f663d120c78e225976d6cd07b2f93a5c5e13012d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1998142f4..008a1f4da 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 81b48e83ecb228712890d48658b212c90441d7d7`. The canonical machine baseline + f663d120c78e225976d6cd07b2f93a5c5e13012d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/general/review/guides/review-execution-contract.md b/knowledge/general/review/guides/review-execution-contract.md index 7d735454a..f3ce6201d 100644 --- a/knowledge/general/review/guides/review-execution-contract.md +++ b/knowledge/general/review/guides/review-execution-contract.md @@ -1,10 +1,10 @@ --- title: "独立审查执行合同" created: 2026-07-13 -updated: 2026-07-31 +updated: 2026-09-02 type: guide tags: [general, review] -sources: ["InferMatrixCopilot Issue #17", "InferMatrixCopilot Issue #24", "zuiho-kai/claude-workflow-starter@c217fc6"] +sources: ["InferMatrixCopilot Issue #17", "InferMatrixCopilot Issue #24", "vllm-project/vllm-omni PR #5871", "zuiho-kai/claude-workflow-starter@c217fc6"] --- # 独立审查执行合同 @@ -46,6 +46,12 @@ sources: ["InferMatrixCopilot Issue #17", "InferMatrixCopilot Issue #24", "zuiho CI 默认只作为状态证据；只有首个失败异常与 frozen diff 重叠或阻塞判决时才打开日志。纯文档改动跳过依赖预检和 pytest，只做 diff hygiene、链接/构建检查以及对被引用 live contract 的有界核对。 +远端 head 必须物化为隔离快照或只用 SHA-addressed read；detached worktree 只冻结身份，不是 +安全边界。fork/untrusted head 的 import、测试、build、hook 和 repo-configurable tool 只有在 +无 secret、限制文件系统/网络/资源且用后销毁的 sandbox 中才可执行，否则只做静态读取与已有 CI。 +快照指纹必须覆盖 tracked、index、任务内 untracked 与 ignored bytes，并在每组可能改写文件的验证 +前后及交付前复核；只断言 `HEAD` 或只记录 untracked 文件名不足以证明证据未漂移。^[vLLM-Omni PR #5871] + 验证记录必须包含 `{repo, head_sha, command, result, environment_fingerprint}`。环境指纹至少覆盖依赖锁摘要、Python 和平台；只有依赖指纹匹配才复用环境。head 变化使旧测试结果失效，但依赖没有变化时不重建环境。预检失败就报告具体不兼容项，不继续运行或声称 pytest 已验证。预检通过后，目标测试和静态检查应与源码审查并行，不能等模型审完才启动。每条变化语义已有 finding 或明确 no-issue 结论后停止，不为增加信心继续追加搜索。 ## 完成条件 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6e79d3a5b..91f6b4d02 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 81b48e83` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f663d120` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 15be9afe0..028b13fe2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 81b48e83` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f663d120` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1a86d396c..2bc16d15c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 81b48e83` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f663d120` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a8d2bcf09..356233314 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 81b48e83` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f663d120` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f28e3b07c..31042d793 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 81b48e83` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f663d120` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9c8ea1c20..bcffee4b6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 81b48e83` 验证存在 +- 源码校验：以上路径均已在 `main @ f663d120` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1a509881b..98b1bdc8b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 81b48e83`（2026-08-08）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f663d120`（2026-08-08）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 81b48e83 \ + --to f663d120 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/review/_index.md b/knowledge/repos/vllm-omni/review/_index.md index 60214d8e3..d09fe7dc7 100644 --- a/knowledge/repos/vllm-omni/review/_index.md +++ b/knowledge/repos/vllm-omni/review/_index.md @@ -1,16 +1,30 @@ --- title: "vLLM-Omni 代码审查" created: 2026-07-10 -updated: 2026-07-31 +updated: 2026-09-02 type: index tags: [vllm-omni, review] -sources: [] +sources: ["PR #5871", .claude/skills/review-pr/SKILL.md, .claude/skills/review-pr/references/process/design-contracts.md, .claude/skills/review-pr/references/process/general-checks.md, .claude/skills/review-pr/references/process/review-execution.md, .claude/skills/review-pr/references/process/review-routing.md] --- # vLLM-Omni 代码审查 -默认从 PR title/body 直接进入 component/model owner；changed files 只校验范围。本目录 -只放 vLLM-Omni 特有的审查方法，不再承担 owner 导航。 +PR title/body 只用于初始 census 和导航；必须由 frozen live diff 的 producer→consumer 与 +branch-local routing 决定 component/model owner，changed files 再验证完整范围。本目录只放 +vLLM-Omni 特有的审查方法，不再承担 owner 导航。 + +上游 `main @ f663d120` 新增 `.claude/skills/review-pr/`，是 vLLM-Omni 仓库自带的 Claude +review skill，不是 InferMatrixCopilot 的 skill/runtime 实现，也不能用其 prompt 副本覆盖本知识树规则。 +可复用的仓库合同只有：冻结目标后从 live producer→consumer 选一个 primary module，真实跨边界时 +才加第二 owner，并叠加命中的 feature/evidence checks；branch-local design 页的 draft candidate +只能作问题，除非代码、测试或政策已执行。外部 fork 只有在无 secret、限文件系统/网络/资源的 +disposable sandbox 才运行代码，否则限于 SHA-addressed 静态读取与已有 CI。review 默认只读；提交 +review event、请求 reviewer 或发 `@mention` 都需要分别明确授权并在写入前复核 head。 + +该 skill 的快照合同还覆盖 tracked/index/untracked/ignored bytes，并在验证组和交付前复核；detached +worktree 只冻结身份，不是安全沙箱。使用时仍以目标文件内容和当前仓库政策为准。PR 只报告结构、 +链接、pre-commit 检查和一次 owner-ranking dry run，没有 behavioral agent eval、runtime test 或独立 +review；这些自报结果不能冒充 review skill 的行为正确性证明。 | 具体问题 | 查看哪里 | |---|---| From 10731cb540a89ef3abbeb1713079aa05a2f3ff7e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 19:10:57 +0800 Subject: [PATCH 058/465] knowledge(vllm-omni): sync 593b4045 (PR #5073) Upstream-Commit: 593b4045391b701fa51b90d38c6f176caaba7a74 Upstream-PR: #5073 Knowledge-Delta: stage-scoped Qwen2.5 config and external embedding width --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/rules.md | 20 +++++++++++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 19 +++++++++++++-- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../vllm-omni/models/qwen-omni/_index.md | 6 +++-- .../models/qwen-omni/architecture.md | 24 +++++++++++++++++-- 13 files changed, 73 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 812eeaeed..45c6ae221 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 81b48e83ecb228712890d48658b212c90441d7d7 + previous_audited_sha: f663d120c78e225976d6cd07b2f93a5c5e13012d audited_ref: main - audited_sha: f663d120c78e225976d6cd07b2f93a5c5e13012d + audited_sha: 593b4045391b701fa51b90d38c6f176caaba7a74 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 008a1f4da..36db5bc43 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - f663d120c78e225976d6cd07b2f93a5c5e13012d`. The canonical machine baseline + 593b4045391b701fa51b90d38c6f176caaba7a74`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 91f6b4d02..ad457413e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f663d120` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 593b4045` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index c43422f68..8d1b1883b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-08-05 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/engine/stage_init_utils.py, tests/engine/test_stage_engine_args.py] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py] --- # vLLM-Omni 配置开发门禁 @@ -24,6 +24,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | deploy YAML、`base_config`、pipeline/stage overlay、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a` | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::{StageConfigFactory.create_from_model,StageConfigFactory._merge_cli_overrides}` | | composable strategy、axis、routing、load balancing、`strategy-config` | `composable-strategy`：`CONF-4a` | `vllm_omni/config/composable_parallel/strategy_loader.py::{parse_strategy_specs,load_strategy_specs}` → `translator.py::translate_strategy_stack` → `apply.py::apply_strategy_specs` → `config_factory.py::{StageConfigFactory._apply_strategy_specs,StageConfigFactory._reconcile_strategy_with_cli}` | | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | +| multi-stage HF sub-config、stage quantization view、`hf_config_name` | `stage-model-config`：`VOMNI-CFG-1i` | pipeline stage declaration → `OmniModelConfig::{draw_hf_text_config,get_model_arch_config}` → vLLM quantization selection | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -32,6 +33,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `deploy-memory` | 显存预算、KV pin、多 stage 共卡 | `CONF-1a`, `CONF-2a` | | `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`, `CONF-4b`, `CONF-5a` | | `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a` | +| `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `VOMNI-CFG-0a`, `VOMNI-CFG-0b` | ## 配置归一化与新老路径一致性 @@ -60,6 +62,22 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR config 就无法 `resolve_pipeline_config` 的 callable entry（当前包括 `qwen3_omni_moe`），并非字面 exhaustive；cutover 前每种 resolver variant 都须补 concrete HF-config fixture。^[PR #5678] +### VOMNI-CFG-1i — multi-stage 模型必须显式选择同一份 stage HF config + +- 触发：nested HF config 的模型新增/修改 `hf_config_name`、text config、quantization metadata、 + embedding width，或 pipeline stage 拆分。 +- 强制：每个 stage 的 `hf_config_name` 必须同时驱动 `draw_hf_text_config()`、 + `get_model_arch_config()` 和 input-embedding width 查询，不能让三者各自回退到 root 的默认 + text config。top-level quantization config 继续走通用 vLLM resolver；stage sub-config 与 + ignore/exclude 规则共同决定实际 quantized modules，不增加 model-type string 特判。 +- 禁止：因为 root 默认返回 Thinker 就省略 Talker/Code2Wav 声明；给每个模型复制 quantization + resolver；仅断言 pipeline 字段存在而不追到 quant config、text hidden size 与 runner consumer。 +- 验收：逐 stage 断言选中的 sub-config 与 architecture/quantization view；覆盖一个无 embedding + override 的 AR control，证明 input width 回退到该 stage text hidden size。Qwen2.5-Omni 当前为 + Thinker=`thinker_config`、Talker=`talker_config`、Code2Wav=`thinker_config`，thinker-only 也用 + `thinker_config`。最终 diff 只测 stage 名与 Qwen3 无 override control，未直接单测 Qwen2.5 + Talker overridden-width runner path；该缺口见 [EXEC-1d](../model-executor/rules.md#exec-1d-cross-stage-embedding-buffer-必须按-ingress-width-分配)。^[PR #5073] + ## 部署配置与资源预算 只有 `CONF-数字字母` 是本节可审计规则 ID。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 028b13fe2..798b602a1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f663d120` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 593b4045` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2bc16d15c..1296f7036 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f663d120` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 593b4045` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 356233314..f97d7c0a0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f663d120` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 593b4045` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 41504b4bf..90ad6308f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5074", "PR #5310", "PR #5610", "PR #5777", "PR #5792", "PR #5824", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5777", "PR #5792", "PR #5824", "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -20,6 +20,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | runner `_preprocess`、逐请求 metadata、prefill/decode phase、batch preprocess、MTP | `runner-preprocess`：本页“Runner 到模型的预处理合同” | `vllm_omni/worker/gpu_model_runner.py::{OmniGPUModelRunner._maybe_run_batch_preprocess,_preprocess,_build_model_kwargs_extra,_talker_mtp_forward}` → `vllm_omni/model_executor/models/<命中模型>` consumer | | stage TP/PP/DP、devices、replica、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | +| cross-stage embedding width、pre-projection buffer | `bridge-batch`：`EXEC-1d` | stage HF config → `get_inputs_embeds_size()` → `GPUARModelRunner.inputs_embeds` → model projection | | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | | fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | H3 text encoder fused owner；其他模型先确认目标 main 是否已有 fused parameter | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | @@ -28,7 +29,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru |---|---|---| | `core` | 每次 model-executor 审查 | `EXEC-1a` | | `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | -| `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`, `EXEC-1b`, `EXEC-1c` | +| `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`, `EXEC-1b`, `EXEC-1c`, `EXEC-1d` | | `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`, `EXEC-2b` | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | @@ -160,6 +161,20 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 验收：同 seed 同输出、异 seed 不同输出，batch reorder/compaction 后逐请求结果稳定；全局 RNG 前后相同，计数器证明目标分支实际消费请求参数。 ^[PR #3422] ^[PR #5074] ^[PR #5792] +### EXEC-1d — cross-stage embedding buffer 必须按 ingress width 分配 + +- 触发：stage 的输入 embedding 在 model forward 或 preprocess 内投影，或修改 shared AR runner + 的 `inputs_embeds` buffer。 +- 强制：buffer second dimension 来自 `model_config.get_inputs_embeds_size()`；它表示进入 stage + 时的 pre-projection width，与内部 `hf_text_config.hidden_size` 分开。没有 stage override 时 + helper 必须回退到内部 hidden size，保持其他 AR stage 的既有行为。 +- 禁止：把内部 hidden size 当跨 stage wire shape；反过来用某模型的外部宽度扩大所有 stage； + 只测 config helper 而不覆盖发生宽度差异的正向 runner/model path。 +- 验收：至少一个 ingress≠internal 的正向 case 与一个无 override control，断言真实 buffer shape + 和 projection consumer。Qwen2.5 Talker 当前接收 3584-wide Thinker state，并在 forward 内投影 + 到 896；Qwen3 在 preprocessing 先投影，因此走 hidden-size fallback。PR #5073 的最终单测只有 + Qwen3 control，没有直接钉住 Qwen2.5 正向 buffer，后者仍是回归缺口。^[PR #5073] + ### EXEC-2a — loader 的 dtype 与 config 获取必须显式、最小化 - 触发：模型 loader 构造 text encoder、VAE、transformer 或只读取 checkpoint config。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 31042d793..b1a9b370e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f663d120` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 593b4045` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index bcffee4b6..b6ffe0cba 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f663d120` 验证存在 +- 源码校验：以上路径均已在 `main @ 593b4045` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 98b1bdc8b..cad07707f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f663d120`（2026-08-08）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 593b4045`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to f663d120 \ + --to 593b4045 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index 4f32459fa..6a474fafa 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -4,14 +4,14 @@ created: 2026-07-16 updated: 2026-07-16 type: index tags: [vllm-omni, models, qwen-omni] -sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, docs/design/qwen3_omni_tts_performance_optimization.md] +sources: ["PR #5073", vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/config/pipeline_registry.py, docs/design/qwen3_omni_tts_performance_optimization.md] --- # Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS） - 常见别名：`qwen2_5_omni`、`qwen3_omni_moe`、`qwen3_tts`（家族目录：同一 thinker/ talker/code2wav 谱系的多个 checkpoint/代际，按别名规则共用本目录） -- 源码模型族（`main @ 5c390096` 验证）：`model_executor/models/qwen2_5_omni/`、 +- 源码模型族（`main @ 593b4045` 验证）：`model_executor/models/qwen2_5_omni/`、 `qwen3_omni/`、`qwen3_tts/`；pipeline registry key `qwen2_5_omni`、 `qwen2_5_omni_thinker_only`、`qwen3_omni_moe`（resolver `resolve_qwen3_omni_pipeline`，位于 `models/qwen3_omni/pipeline.py`，由 @@ -19,6 +19,8 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline - deploy YAML：`qwen2_5_omni.yaml`（1×H100 验证）、`qwen3_omni_moe.yaml` （2×H100 验证）、`qwen3_omni_moe_mori_intranode.yaml`、`qwen3_tts.yaml` （+ `_forced_aligner`、`_high_concurrency` 变体） +- Qwen2.5 thinker-only 的 ModelOpt NVFP4 W4A4 checkpoint 支持、stage sub-config、Talker + ingress/internal width 与硬件/accuracy 证据边界归下方 architecture 入口。 - 官方历史文档：`docs/design/module/archive/async_omni_architecture.md`（以 Qwen3-Omni 为 worked example 的分层运行时快照，非 active spec）、 `docs/design/qwen3_omni_tts_performance_optimization.md`（性能优化实录） diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md index 790288ecb..1f0fb2e73 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md @@ -4,12 +4,12 @@ created: 2026-07-16 updated: 2026-07-16 type: architecture tags: [vllm-omni, models, qwen-omni] -sources: [docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/model_executor/models/registry.py] +sources: ["PR #5073", docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/config/model.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/registry.py, vllm_omni/worker/gpu_ar_model_runner.py] --- # Qwen-Omni 家族拓扑与性能结论 -以下事实在 `main @ 5c390096` 复核。 +以下事实在 `main @ 593b4045` 复核。 ## Stage 拓扑 @@ -26,6 +26,26 @@ sources: [docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/mo resolve_qwen3_omni_pipeline`）按 checkpoint 结构动态决定，而不是冻结的 `PipelineConfig` 字面量。 +## Qwen2.5-Omni ModelOpt NVFP4 边界 + +- `qwen2_5_omni` full pipeline 和 `qwen2_5_omni_thinker_only` 的 Thinker 都显式用 + `thinker_config`；full Talker 用 `talker_config`，Code2Wav 用 `thinker_config`。同一个 + `hf_config_name` 同时选择 stage text config、architecture/quantization view 与 input width。 +- thinker-only ModelOpt NVFP4 W4A4 不增加 kernel 或 Qwen2.5 专用 resolver：checkpoint root 的 + `quantization_config` 进入 vLLM 既有 `ModelOptNvFp4Config`，ignore/exclude 保持 audio/visual + encoder、embedding、LM head、Talker、Token2Wav/Code2Wav 和 KV cache 为 BF16；只有 Thinker + language-backbone linear 命中现有 FlashInfer Cutlass NVFP4 kernel。支持声明因此绑定 ModelOpt + checkpoint metadata 与排除规则，不能泛化成所有 Qwen2.5 stage 或任意 FP4 checkpoint。 +- Qwen2.5 Talker 的外部 3584 width 在 forward 内投影到内部 896；shared runner 的 buffer 必须按 + `get_inputs_embeds_size()` 分配。Qwen3 在 preprocessing 已投影，未 override 时仍回退 internal + hidden size。共享门禁见 [EXEC-1d](../../components/model-executor/rules.md#exec-1d-cross-stage-embedding-buffer-必须按-ingress-width-分配)。 +- 最终 head 的外部验证只覆盖单张 RTX PRO 6000 Blackwell、TP1、特定 vLLM/PyTorch/ + Transformers/FlashInfer 栈：full three-stage server 启动、仅 Thinker 选 NVFP4 kernel，以及 + deterministic text smoke。accuracy/perf 也是同一硬件上的 text-output workload，且相对 BF16 + accuracy 分别下降 1.17 与 3.5 percentage points；这些是 case-bound 取舍，不是质量或加速保证。 + 更早的表来自旧 commit backport/旧依赖，不能当最终 head 证据。最终 diff 也缺 Qwen2.5 + ingress≠internal 的直接 unit test。^[PR #5073] + ## 官方性能优化结论（docs/design/qwen3_omni_tts_performance_optimization.md） 两模型共享同一多 stage 设计，叠加同一组优化：**batching**（逐 stage 提升 GPU From f03d36547b63237e8cd9efd8877ca36845387c7d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 19:17:32 +0800 Subject: [PATCH 059/465] knowledge(vllm-omni): sync 9159cedb (PR #5914) Upstream-Commit: 9159cedbd7ce71e8d29899867e68524ead6c03bc Upstream-PR: #5914 Knowledge-Delta: stage-based architecture overview and live-boundary caveats --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../components/configuration/architecture.md | 8 +++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/docs/design-doc-map.md | 16 +++++++++++++--- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../repos/vllm-omni/models/minimax-h3/_index.md | 7 ++++--- 12 files changed, 35 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 45c6ae221..77f96dc17 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f663d120c78e225976d6cd07b2f93a5c5e13012d + previous_audited_sha: 593b4045391b701fa51b90d38c6f176caaba7a74 audited_ref: main - audited_sha: 593b4045391b701fa51b90d38c6f176caaba7a74 + audited_sha: 9159cedbd7ce71e8d29899867e68524ead6c03bc # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 36db5bc43..22e5f8d81 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 593b4045391b701fa51b90d38c6f176caaba7a74`. The canonical machine baseline + 9159cedbd7ce71e8d29899867e68524ead6c03bc`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ad457413e..5df505973 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 593b4045` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9159cedb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/architecture.md b/knowledge/repos/vllm-omni/components/configuration/architecture.md index 47efd0ed7..0e211acdd 100644 --- a/knowledge/repos/vllm-omni/components/configuration/architecture.md +++ b/knowledge/repos/vllm-omni/components/configuration/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: architecture tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #5647", "PR #5678", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/diffusion/data.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/entrypoints/cli/serve.py] +sources: ["claude-workflow-starter-private@296ea45", "PR #5647", "PR #5678", "PR #5914", docs/design/architecture_overview.md, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/diffusion/data.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/entrypoints/cli/serve.py] --- # vLLM-Omni 配置构造架构 @@ -66,6 +66,12 @@ structured 与 legacy 可以有不同的最终对象，但不能有不同的字 cutover 前的兼容适配器和测试 oracle，不是当前 live startup source of truth；目标 pin 中 typed builder 与 `StageConfigFactory.create_from_model` 都没有 non-test caller。 +`docs/design/architecture_overview.md` 的五层图是目标架构的 conceptual map，不改变上述 live +边界。图中的 authoring→resolve→transport-safe config→launch plan→engine materialization 可用于 +定位 owner，但 target `9159cedb` 仍不能据此宣称 structured typed startup 已完成 production +cutover；review 后的文档使用真实 `BaseVllmOmniStageConfig`、`StageConfigFactory.create_from_model()` +与 `VllmOmniConfig.from_pipeline_config()` 名称，不存在额外 resolver API。^[PR #5914] + ## 值状态合同 | 状态 | 必须怎样处理 | diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 798b602a1..c77f70fb9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 593b4045` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9159cedb` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1296f7036..e9e1a31f9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 593b4045` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9159cedb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f97d7c0a0..12cf3b8d9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 593b4045` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9159cedb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b1a9b370e..abd9dc6e6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 593b4045` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9159cedb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b6ffe0cba..c61561c61 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 593b4045` 验证存在 +- 源码校验：以上路径均已在 `main @ 9159cedb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index 74da5da21..1d89debb1 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -4,12 +4,12 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, docs] -sources: ["PR #5139", "PR #5833", "PR #5839", docs/.nav.yml, docs/design/index.md, docs/design/module/, docs/design/module/archive/README.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] +sources: ["PR #5139", "PR #5833", "PR #5839", "PR #5914", docs/.nav.yml, docs/design/index.md, docs/design/module/, docs/design/module/archive/README.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] --- # 官方设计文档地图（docs/design/**） -`main @ fd0e9218` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 按 architecture、feature、 +`main @ 9159cedb` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 按 architecture、feature、 infrastructure/performance 和 21 个 module 页面编排。module frontmatter 的 owner、代码路径、依赖与 验证路径可作 review 首跳 source map；但本批页面全部是 `status: draft`，其中 invariant 编号也明确 只是 candidate，不能覆盖当前代码与测试。尤其 `module/vllm_omni_config.md` 是 @@ -25,9 +25,19 @@ infrastructure/performance 和 21 个 module 页面编排。module frontmatter 尚未支持，placement-group/actor helpers 也没有 live caller，不能据残留宣称 runtime support。 `docs/design/ar_diffusion_pipeline_capability.md` 仍未列入索引，找 spec 时还须检查实际文件树。 +#5914 将 overview 从 `OmniRouter` 和三类 AR/DiT 图改为 stage-based 地图：四个代表模型、指标词汇、 +engine/orchestrator/runtime/connector ownership、五层 config/runtime resolution、feature 与 public +interface。它是跨 owner 导航，不是新 runtime 支持证据：MiniMax-H3 的 encoder/DiT/VAE 仍是同一 +registered diffusion stage 内的 component/phase，独立 placement 需要未来 multi-stage pipeline； +Cosmos3 reasoner/generator 同样在一个 diffusion stage。overview 的 typed config 图是 conceptual； +目标代码中 `create_from_model()` 委托 `VllmOmniConfig.from_pipeline_config()`，但稳定生产 startup +仍走 legacy stage 构造，所以不能据图宣称 typed cutover 完成。duplex 也仍是 experimental。 +PR 只报告 strict MkDocs build（仍有既存 navigation/link warnings）与 diff hygiene，没有 runtime +测试；代表模型的 batching/attention/parallelism/quantization 表须回各 live owner 验证。^[PR #5914] + | 官方文档 | 一句话 | 知识树 owner | |---|---|---| -| `architecture_overview.md` | 三类模型拓扑（DiT 主/AR 主/AR+DiT）、OmniRouter/EntryPoints/AR/Diffusion/OmniConnector、E/P/D/G 解耦、CFG companion 流 | [serving](../components/serving/architecture.md) + [distributed](../components/distributed/architecture.md) | +| `architecture_overview.md` | stage 定义、Qwen3-Omni/HunyuanImage-3.0/MiniMax-H3/Cosmos3 代表映射、指标词汇、system/config/interface 概念图；具体 support 仍回 live owner | [serving](../components/serving/architecture.md)、[configuration](../components/configuration/architecture.md)、[distributed](../components/distributed/architecture.md) + [Qwen](../models/qwen-omni/_index.md)/[Hunyuan](../models/hunyuan-image3/_index.md)/[MiniMax](../models/minimax-h3/_index.md)/[Cosmos3](../models/cosmos3/_index.md) | | `module/{entrypoints,vllm_omni_config,input_output_modality_contracts,error_contracts,engine_orchestration,stage_runtime,omni_connector,model_integration,ar_runtime}.md` | serving/config/I-O/error/engine/stage/connector/model/AR 的 draft ownership 与 source map；candidate invariant 尚非规范 | 对应 [serving](../components/serving/_index.md)、[configuration](../components/configuration/_index.md)、[distributed](../components/distributed/_index.md)、[scheduler](../components/scheduler/_index.md)、[model-executor](../components/model-executor/_index.md) | | `feature/omni_async_output_materialization.md` | AR decode 关键路径外的异步 D2H 与 Omni payload 构造；含 live connector ownership、平台验证范围和同步 fallback | [model-executor](../components/model-executor/_index.md) | | `module/diffusion/{index,diffusion_runtime,diffusion_model_integration,continuous_batching,parallelism,offloader}.md` | diffusion draft module 边界与首跳路径 | [diffusion](../components/diffusion/architecture.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index cad07707f..3f130cce1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 593b4045`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9159cedb`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 593b4045 \ + --to 9159cedb \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 8fe6ccafe..025f6450f 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5914", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -22,9 +22,10 @@ confidence: high 选择 FL2VA-only，`ref2va` 选择 Ref2VA-only。combined 省略 request task 时，无媒体→t2va、 image-only→fl2va、video/audio→ref2va（因此 image-only Ref2VA 必须显式 task）；Ref2VA-only 省略时固定→ref2va。 -- 输出是带同步音频的 MP4；FL2VA 接受首帧、尾帧或有序首尾帧，Ref2VA 支持 image-only +- pipeline 边界输出 decoded frames、stereo audio tensor、FPS 与 sample rate；serving/example + output layer 才负责编码 H.264 和 mux 成带同步音频的 MP4。FL2VA 接受首帧、尾帧或有序首尾帧，Ref2VA 支持 image-only 及有 visual reference 的 image/video/audio mixed matrix。完整计数、媒体和 API 合同见 - MMH3-2a/2b。 + MMH3-2a/2b。^[PR #5914] - 生成合同固定为 24 FPS 视频与 32 kHz 音频；官方输出 duration、named ratio、768 short-edge 与 32-pixel canvas policy 在 request validation 阶段执行，而不是由 VAE 静默修正。 From c3903a2fc91c77bfcaf955a2a94e8024d76ad23b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 19:25:19 +0800 Subject: [PATCH 060/465] knowledge(vllm-omni): sync 8e2e9b6b (PR #4657) Upstream-Commit: 8e2e9b6b53e86e6a479ed2c0a53782f655f60e04 Upstream-PR: #4657 Knowledge-Delta: Cosmos3 request-scoped UND session state and memory boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 5 ++- .../components/diffusion/session-state.md | 37 +++++++++++++++++++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +- .../repos/vllm-omni/models/cosmos3/_index.md | 9 ++++- .../repos/vllm-omni/models/cosmos3/rules.md | 19 +++++++++- 12 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/session-state.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 77f96dc17..c5b999ff2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 593b4045391b701fa51b90d38c6f176caaba7a74 + previous_audited_sha: 9159cedbd7ce71e8d29899867e68524ead6c03bc audited_ref: main - audited_sha: 9159cedbd7ce71e8d29899867e68524ead6c03bc + audited_sha: 8e2e9b6b53e86e6a479ed2c0a53782f655f60e04 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 22e5f8d81..cd6fd98d6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 9159cedbd7ce71e8d29899867e68524ead6c03bc`. The canonical machine baseline + 8e2e9b6b53e86e6a479ed2c0a53782f655f60e04`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5df505973..62caa1755 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9159cedb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8e2e9b6b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c77f70fb9..e222f5468 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -1,7 +1,7 @@ --- title: "Diffusion" created: 2026-07-10 -updated: 2026-08-23 +updated: 2026-09-02 type: index tags: [vllm-omni, components, diffusion] sources: [] @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9159cedb` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8e2e9b6b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -36,3 +36,4 @@ sources: [] | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | +| 实验性 world-model session 生命周期、LRU 与内存统计边界 | [session state](session-state.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/session-state.md b/knowledge/repos/vllm-omni/components/diffusion/session-state.md new file mode 100644 index 000000000..08499d40c --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/session-state.md @@ -0,0 +1,37 @@ +--- +title: "World-model session state" +created: 2026-09-02 +updated: 2026-09-02 +type: architecture +tags: [vllm-omni, components, diffusion] +sources: ["PR #4657", docs/features/session_state_manager.md, vllm_omni/experimental/world_models/session_state/, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py] +confidence: high +--- + +# World-model session state + +实验性 session-state manager 只负责 request/session 生命周期、LRU 表和存储统计；具体 +state 的形状、键和装卸语义由 model adapter 拥有。不要把 Cosmos3 的 dense UND encode-once +K/V 与 [DreamZero](../../models/dreamzero/architecture.md) 的 paged AR-diffusion KV 抽象成同一 +缓存合同。 + +## 生命周期与内存边界 + +- session ID 选择 manager entry；何时 reset/drop、是否跨 request 保留由 model adapter 与 + pipeline 定义，不能从 manager 本身推断 session 的持久性。 +- `max_sessions` 是 entry 数量 LRU：淘汰只移除 manager lookup，活动 adapter 或 transformer + alias 仍可保留对象和底层 storage，因此它不是 OOM 上界。 +- `byte_budget` 当前只记录、不执行。stats 按 device 对物理 storage 去重，但 manager 外的 + alias（例如 drop 后 transformer 最后加载的 K/V）不会计入，可能低报进程实际驻留。 +- adapter 隔离了已存 state，却没有让共享 mutable pipeline 并发安全；transformer 的 + load→forward handoff 仍是可变状态区间。 + +## Cosmos3 接入与证据 + +Cosmos3 在每次 generation 开始 reset，并让整个 denoise dispatch 经 `finally` drop；因此该 +接入是 request 内临时缓存，不是跨请求记忆。其 branch/layer key、RoPE 恢复、CFG 形态和 +fail-closed 条件见 +[COSMOS-2c](../../models/cosmos3/rules.md#cosmos-2c-session-manager-缓存必须按-branchlayer-完整装卸)。 +功能默认关闭，保留原 transformer-instance cache。PR #4657 的 CPU adapter 与 mocked +pipeline 测试覆盖生命周期和路由，但没有真实 GPU/model 的 flag-on 集成、当前 head +equivalence 或 peak-memory A/B；早期 `MAX_ABS_DIFF=0` 使用了错误环境变量，不能作为证据。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e9e1a31f9..d54da6842 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9159cedb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8e2e9b6b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 12cf3b8d9..4c6c46c73 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9159cedb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8e2e9b6b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index abd9dc6e6..a3b12427d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9159cedb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8e2e9b6b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c61561c61..cfdff84c2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9159cedb` 验证存在 +- 源码校验：以上路径均已在 `main @ 8e2e9b6b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3f130cce1..5df70bbd3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9159cedb`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8e2e9b6b`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 9159cedb \ + --to 8e2e9b6b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/_index.md b/knowledge/repos/vllm-omni/models/cosmos3/_index.md index feb68425e..aa09cfa9e 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/_index.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5001", "PR #5634", recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/platforms/rocm/platform.py] +sources: ["PR #4657", "PR #5001", "PR #5634", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py] confidence: high --- @@ -38,6 +38,13 @@ guardrails off 的一次 warmup + 一次测量，不是当前 pin 或通用 ROCm platform 的 AITER capability gate，未在该 PR 实测；质量、multi-GPU 与 Cosmos3-Super 也未覆盖。 引用具体数字或扩展支持矩阵时按本页审查入口中的 `COSMOS-3b` 复核。 +## Session-state opt-in + +实验 flag 可把 UND text K/V 与分支 `freqs_gen` 接入共享 manager；默认关闭，并且只在一次 +request 的 denoise 生命周期内使用，不提供跨请求记忆或并发安全。具体 branch/layer 合同见 +本页审查入口中的 `COSMOS-2c`，共享 LRU、 +统计与引用存活边界见 [world-model session state](../../components/diffusion/session-state.md)。 + ## 什么时候查这里 - 审查 Cosmos3 Edge/Distilled scheduler、RNG、guidance、offload 或 capability claim。 diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index c6ecb473a..4edc40a8f 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5001", "PR #5634", recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/platforms/rocm/platform.py] +sources: ["PR #4657", "PR #5001", "PR #5634", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py] confidence: high --- @@ -20,6 +20,7 @@ confidence: high | Distilled、SDE scheduler、`t_list` | COSMOS-1b | `diffusion/models/cosmos3/pipeline_cosmos3.py::Cosmos3OmniDiffusersPipeline` 及 scheduler config consumer | | seed/generator、逐步噪声 | COSMOS-2a | `pipeline_cosmos3.py::Cosmos3OmniDiffusersPipeline` 的 request-local sampling 路径 | | `guidance=0`、request extra | COSMOS-2b | `model_extras/cosmos3.py` → pipeline sampling params | +| session manager、UND K/V、`freqs_gen`、CFG branch、request ID | COSMOS-2c | `state_cosmos3_adapter.py` → `pipeline_cosmos3.py` → `transformer_cosmos3.py` | | online/offload/HSDP/VAE parallel 支持声明 | COSMOS-3a | capability 文档/recipe → 对应公开入口与实现路径 | | ROCm、MI350X、AITER、latency/peak memory | COSMOS-3b | recipe 的 measurement commit/protocol → `platforms/rocm/platform.py` backend gate → 目标配置复测 | @@ -58,6 +59,22 @@ confidence: high - 禁止：`value or default`、truthy sentinel。 - 验收：未提供使用默认，`0.0` 保持零，普通正值保持不变，三类测试都到 consumer。 ^[PR #5001] +## COSMOS-2c — session manager 缓存必须按 branch/layer 完整装卸 + +- 触发：启用 session-state manager，或修改 UND cached K/V、`freqs_gen`、CFG/request key。 +- 强制：每层以 conditional `und_kv/{i}` 或 unconditional `und_kv_neg/{i}` 分支存 native + `(K,V)` 引用，并随分支保存 M-RoPE cos/sin；仅未初始化分支可 capture，标准与 RoboLab/action + 路径都必须传 request ID。generation 开始 reset，整个 dispatch 用 `finally` drop session。 +- 禁止：capture 新分支时接受缺 K/V 或缺 freqs，或 load 只初始化部分 layer K/V；已有完整 + K/V 但没有 cached freqs 时允许装载，并由 transformer 独立重算 freqs。flag 开启时 + `diffuse_transfer()` 必须 fail closed，`_bde_kv_state` 非空也必须拒绝，因为当前没有 + writer。默认关闭时保留原有 transformer-instance cache 行为。^[PR #4657] +- CFG：sequential CFG 可在同 GPU 保留 cond+uncond，CFG-parallel 每 rank 只保留自身分支， + no-CFG 只保留 cond；UND K/V 随 SP rank 复制。该 state 隔离不等于 pipeline 并发安全。 +- 验收：CPU adapter/mocked pipeline 测试覆盖 branch round-trip、encode-once、freqs restore、 + partial rejection、reset/LRU pin、RoboLab key、Transfer guard 与首次 step 前异常清理；真实 + GPU/model flag-on equivalence 和 peak-memory A/B 仍是缺口，错误 env var 下的早期零差异不能引用。 + ## COSMOS-3a — 支持表中的每个能力都有独立证据 - 触发：声明 Edge/Distilled、online、offload、HSDP、VAE parallel 或公开 checkpoint 支持。 From a902b93390fe8b320707dd5c9ce0f271e79ecf13 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 19:31:57 +0800 Subject: [PATCH 061/465] knowledge(vllm-omni): sync ebe93f5d (PR #5682) Upstream-Commit: ebe93f5d1e794202598e652a45fa74a29a7f698b Upstream-PR: #5682 Knowledge-Delta: adapter-owned TTS detection and migration ratchet --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../configuration/adding-a-model.md | 16 +++++++++--- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules.md | 25 +++++++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- 11 files changed, 47 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c5b999ff2..8302bcc99 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9159cedbd7ce71e8d29899867e68524ead6c03bc + previous_audited_sha: 8e2e9b6b53e86e6a479ed2c0a53782f655f60e04 audited_ref: main - audited_sha: 8e2e9b6b53e86e6a479ed2c0a53782f655f60e04 + audited_sha: ebe93f5d1e794202598e652a45fa74a29a7f698b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index cd6fd98d6..32f0d6790 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 8e2e9b6b53e86e6a479ed2c0a53782f655f60e04`. The canonical machine baseline + ebe93f5d1e794202598e652a45fa74a29a7f698b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 62caa1755..6eede31fe 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8e2e9b6b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ebe93f5d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/adding-a-model.md b/knowledge/repos/vllm-omni/components/configuration/adding-a-model.md index 6acbbacbe..9fdaa5dd2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/adding-a-model.md +++ b/knowledge/repos/vllm-omni/components/configuration/adding-a-model.md @@ -1,15 +1,15 @@ --- title: "vLLM-Omni 新模型接入路径" created: 2026-07-16 -updated: 2026-07-29 +updated: 2026-09-02 type: guide tags: [vllm-omni, components, config, models] -sources: ["claude-workflow-starter-private@296ea45", docs/contributing/model/] +sources: ["PR #5682", "claude-workflow-starter-private@296ea45", .claude/skills/add-tts-model/SKILL.md, docs/contributing/model/, vllm_omni/entrypoints/openai/tts_adapters/] --- # 加新模型：三条官方路径与四个注册点 -官方 spec（`main @ 5c390096` 复核）： +官方 spec（`main @ ebe93f5d` 复核）： `docs/contributing/model/adding_omni_model.md`（多 stage omni，以 Qwen3-Omni 为 完整示例——目录结构/关键组件/注册/stage 配置/stage input processor/测试/recipe 九节）、`adding_diffusion_model.md`（纯 diffusion pipeline）、 @@ -27,6 +27,16 @@ sources: ["claude-workflow-starter-private@296ea45", docs/contributing/model/] 另有：跨 stage 转换 `stage_input_processors/.py`（`ar2diffusion` 等）、 serving TTS 适配 `entrypoints/openai/tts_adapters/`、`recipes//.md`。 +## TTS serving 适配 + +AR TTS 新模型写一个 adapter，并在 `tts_adapters/__init__.py` 底部 import 注册；不要在 +`serving_speech.py` 复制 stage 集合或增加 model-type branch。通常声明 `stage_keys`；stage +歧义/缺失时才声明 `model_archs`，只有无专有 stage 的 AR entry 才设 +`arch_identifies_entry_stage`。非集合匹配用 `matches()`，部署拓扑能力用 +`stage_serves_speech()`，真实冲突才设 `detect_priority`。详细 detection 不变量见 +[SERV-5e](../serving/rules.md#serv-5e-tts-detection-从-adapter-metadata-的有序并集派生)。纯 +diffusion TTS 仍走 `for_diffusion()` 直达路径，不能假设 adapter 的 `validate()`/`build()` 可达。 + ## 照抄谁 参照清单见 [reference-models](../../models/reference-models.md)（GLM-Image / diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e222f5468..93b1388c3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8e2e9b6b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ebe93f5d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d54da6842..709744ce4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8e2e9b6b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ebe93f5d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4c6c46c73..1ae095ddb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8e2e9b6b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ebe93f5d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a3b12427d..3ffe01b80 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8e2e9b6b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ebe93f5d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cfdff84c2..d6f9c0344 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8e2e9b6b` 验证存在 +- 源码校验：以上路径均已在 `main @ ebe93f5d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 3d4381f05..9d278d3c8 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5713", "PR #5732", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] confidence: high --- @@ -25,6 +25,7 @@ confidence: high | endpoint restriction、route/app-state guard、capability、公开 400 | `endpoint-capability`：`SERV-4c`, `SERV-4d`, `SERV-5d` | endpoint policy → `api_server.py::build_app` assembled app → public handler | | sleep/wake、partial stage/tag、idempotency、ACK、generation admission | `engine-lifecycle`：`SERV-5a`, `SERV-5b` | `entrypoints/async_omni.py::{AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni.generate}` → `worker/base.py::{handle_sleep_task,handle_wake_task}` / `diffusion/worker/diffusion_worker.py` | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | +| TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | | video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | @@ -39,7 +40,7 @@ confidence: high | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c` + 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d`, `SERV-5d` | -| `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵 | `SERV-5a`, `SERV-5b`, `SERV-5c` | +| `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection | `SERV-5a`, `SERV-5b`, `SERV-5c`, `SERV-5e` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6c` | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`, `SERV-4b`, `SERV-4c`, `SERV-4d`, `SERV-4e`, `SERV-4f`, `SERV-4g`, `SERV-4h` | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`, `SERV-4j`, `SERV-4k` | @@ -277,6 +278,26 @@ confidence: high - 验收：把 handler 置空、移除 route/method、绕过 inner app 三类 mutation 均使 guard 失败； 正常 app 的 HEAD/OPTIONS census 与必需 state key 完整。 ^[PR #6202] +### SERV-5e — TTS detection 从 adapter metadata 的有序并集派生 + +- 触发：增加/迁移 TTS adapter，修改 `stage_keys`、`model_archs`、stage discovery、检测顺序 + 或 speech-capable deployment topology。 +- 强制：adapter 与显式 `LEGACY_TTS_DETECTORS` 构成完整 detector 并集；按 + `(detect_priority, name)` 确定性排序，等优先级 detector 不得重叠。stage-key 候选、仅靠 + architecture 定位的 AR entry stage、最终 model type 都从该并集派生；非集合规则覆写 + `matches()`，拓扑能力覆写 `stage_serves_speech()`，不要回填 `serving_speech.py` 分支。 +- 边界：VoxCPM2 talker architecture 是高优先级权威匹配；Ming flash 的 `ming_tts` legacy + stage 先于 Ming dense 的 architecture fallback；CoVo 的通用 `fused_thinker_talker` 还需 + architecture 确认；Audex omni 还需同部署的 `audex_code2wav`。纯 diffusion 的 + `for_diffusion()` 直接绕过 adapter detection，当前 `DiffusionTTSAdapter` 没有生产 subclass。 +- 验收：冻结旧 ladder 作为 oracle，覆盖 in-tree pipeline stage、adapter 声明 architecture、 + stage 集合等价、entry-arch 范围、无歧义/稳定顺序和 topology guard。`LegacyDetector` 只能缩减； + AST ratchet 的 branch/legacy budget 也只能下降，但 table dispatch 是已固定的漏检形态，不能把 + pre-commit 通过解释为不可绕过。`omnivoice_generator` 未被 pipeline 发出，是保留兼容事实而非 + 可达性证据。^[PR #5682] +- 证据：CPU oracle/ratchet suite 支撑检测等价；L20X VoxCPM2 只证明 architecture path 到达 + warmup，随后因基线同样复现的 vLLM skew 退出，未证明成功 speech endpoint/audio E2E。 + ## Full-duplex 与 CFG companion 生命周期 ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5df70bbd3..bbeb183ad 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8e2e9b6b`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ebe93f5d`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 8e2e9b6b \ + --to ebe93f5d \ --repo \ --mode report-only ``` From cc652149cbdb304d4c50ba9cce4d81f3d3d04047 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 19:42:34 +0800 Subject: [PATCH 062/465] knowledge(vllm-omni): sync 59263701 (PR #5896) Upstream-Commit: 592637015ffccec514401b700548768232d9590c Upstream-PR: #5896 Knowledge-Delta: MindIE RoPE batch-rank normalization for MiniMax-H3 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/rules.md | 9 ++++++--- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../repos/vllm-omni/models/minimax-h3/_index.md | 5 +++-- .../repos/vllm-omni/models/minimax-h3/rules.md | 14 ++++++++++---- 12 files changed, 30 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8302bcc99..ed38cfee6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8e2e9b6b53e86e6a479ed2c0a53782f655f60e04 + previous_audited_sha: ebe93f5d1e794202598e652a45fa74a29a7f698b audited_ref: main - audited_sha: ebe93f5d1e794202598e652a45fa74a29a7f698b + audited_sha: 592637015ffccec514401b700548768232d9590c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 32f0d6790..b72d9fe15 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - ebe93f5d1e794202598e652a45fa74a29a7f698b`. The canonical machine baseline + 592637015ffccec514401b700548768232d9590c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6eede31fe..beabec34f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ebe93f5d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 59263701` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 93b1388c3..acdd26d36 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ebe93f5d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 59263701` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 9bac797ab..c48e268e1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py] confidence: high --- @@ -84,12 +84,15 @@ async output readiness、per-worker result channel 与 shutdown 合同见 - 强制：把 parameter 存储 dtype 与 reduction/累加 dtype 分开约束；CUDA/HIP eager 可尝试 fused op，但异常必须回退到数值等价的 native 实现，`torch.compile` tracing 直接走 native， NPU/MindIE 等专用 backend 则按其真实参数与频率布局调用。RoPE 适配器必须显式区分 full-dim - tiled frequency 与 fused kernel 的 half layout，并保持未旋转 head tail 不变。 + tiled frequency 与 fused kernel 的 half layout，并保持未旋转 head tail 不变。若 caller 可传 + 无 batch 的 `[S,H,D]`，进入只接受 `[B,S,H,D]` 的 fused kernel 前必须补 batch 维，返回后仅在 + 本次确实补维时移除；原生 4D 输入不得被 squeeze。 - 禁止：用 platform dispatch 推断所有 backend 接收同一布局；用 CPU/reference 或 mock 参数 wiring 测试声称真实 CUDA/HIP/NPU fused kernel 已完成数值 parity。 - 验收：native reference 覆盖 parameter dtype、FP32 accumulation、旋转维度与 untouched tail； compile 路径断言不调用 fused op，平台 mock 只作为 wiring 证据。声称 fused parity 还需目标 - 硬件上的同输入数值测试。^[PR #5801] + 硬件上的同输入数值测试。mock kernel 的维度断言与 identity 回传只证明 adapter wiring/shape， + 不证明实际旋转数值。^[PR #5801] ^[PR #5896] ## Checkpoint 与分布式加载 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 709744ce4..ccd949bce 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ebe93f5d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 59263701` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1ae095ddb..beb5d348d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ebe93f5d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 59263701` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3ffe01b80..baf46e245 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ebe93f5d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 59263701` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d6f9c0344..83f150d1a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ebe93f5d` 验证存在 +- 源码校验：以上路径均已在 `main @ 59263701` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index bbeb183ad..306c08fef 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ebe93f5d`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 59263701`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to ebe93f5d \ + --to 59263701 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 025f6450f..f1b94a9c5 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5914", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5896", "PR #5914", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -56,7 +56,8 @@ confidence: high 另报告 Ascend NPU smoke/FL2VA 成功。目标实现接纳已注册的非 CPU device module，但持续 回归、XPU/ROCm 与并发边界仍见 MMH3-2c。 - H3 q/k 使用共享 RMSNorm（BF16 gamma、native FP32 accumulation）与 NeoX RoPE；每个 - 128 维 head 只旋转前 96 维并保留后 32 维，平台布局和验证边界见 MMH3-1c。 + 128 维 head 只旋转前 96 维并保留后 32 维；MindIE-SD 前还要把 H3 的 3D q/k 临时补成 + 4D batch layout，再恢复原 shape。平台布局和验证边界见 MMH3-1c。 - Ascend NPU 可选择 RainFusion 稀疏 video tail，并从 BF16 checkpoint 做 online INT8；两者只在 exact T2VA/Ulysses/no-AllGather DLO 配置有完成证据，几何 fallback、TP width 与组合边界见 MMH3-1a/1d。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 86cad6e73..10c8163e7 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5853", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5853", "PR #5896", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] confidence: high --- @@ -58,13 +58,18 @@ confidence: high native，compile tracing 直接 native；XPU 走 native，NPU 直接调用 `npu_rms_norm` 且没有该 fallback。RoPE 使用 NeoX `RotaryEmbedding(half_head_dim=False)`：128 维 head 只旋转前 96 维 并原样保留末 32 维；CUDA/HIP/native 把 H3 的 tiled full frequencies 转为 half kernel layout， - XPU/MUSA 明确走 native，MindIE 接收 full layout。 + XPU/MUSA 明确走 native，MindIE 接收 full layout。H3 `_apply_rope` 的 q/k 是 3D `[S,H,D]`； + MindIE-SD kernel 只收 4D，因此 shared adapter 必须临时变为 `[1,S,H,D]` 并把输出恢复到 3D。 - 禁止：把任意 96 维随机 frequency 当成 H3 producer 合同；合法 full layout 是同一 48 维 half 的拼接。也不能从 CPU/reference 与 mocked NPU/MindIE wiring 推断真实 fused-kernel parity。 + PR #5896 的新增 mock 只断言 MindIE 收到 4D、输出恢复 3D 且 identity 回传不变；没有执行真实 + rotary 数学，也没有新增原生 4D control。 - 验收：以 `cat([half, half])` frequency 对照 native reference，逐值断言 96 维旋转和 32 维 passthrough；另测 BF16 gamma/FP32 accumulation、compile native path 与各平台参数 wiring。 真实 kernel 性能与数值结论需目标硬件复测；PR 报告的 491014→475070 ms 来自非目标 - `5215e03a` 且缺硬件与重复次数，只能作为有界外部观察，不能写成稳定 3.35% 保证。^[PR #5801] + `5215e03a` 且缺硬件与重复次数，只能作为有界外部观察，不能写成稳定 3.35% 保证。PR #5896 + body 报告 8×Ascend 910B3、USP8/Ring1/DLO、50-step T2VA 完成并附视频，但 commit 只写 + `main`、无数值对照或重复测量；它只补充运行完成证据。^[PR #5801] ^[PR #5896] ## MMH3-1d — RainFusion 只在完整、对齐的 H3 video tail 上稀疏 @@ -302,7 +307,8 @@ confidence: high 必须走 `video_reference` data URL，使 server 持久化 `source_path` 给 ffprobe/ffmpeg consumer。 缺 ffmpeg 时 fallback `mp4v`，但 H3 要求 H.264/H.265，因此该 fallback 不是有效 H3 兼容路径。 - 禁止：把上述 predecessor 数字当成 final repo-root modular route 的实测，或泛化到 50-step、 - 并发及其他 topology；final diff 也没有 PR body 所述 RoPE 代码修复。特殊 payload + 并发及其他 topology；PR #5836 final diff 本身没有其 body 所述 RoPE 修复，后续 PR #5896 + 才补 MindIE 3D→4D adapter。特殊 payload 当前靠 resolved model 字符串包含大小写敏感的 `MiniMax-H3` 判断；materialized env/cache path 若不含该 token 会退回 multipart `input_reference`，所以 lane 接入本身不证明 V2V 稳定执行。 - 验收：nightly 必须证明四个 pytest case 均实际 collected、3/3 completed、artifact/log 上传， From 025d117fcd18d8e391bca88fa339b6ce9cd62ec6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 19:50:36 +0800 Subject: [PATCH 063/465] knowledge(vllm-omni): sync f01e4bdd (PR #5723) Upstream-Commit: f01e4bdddcb90193119f4fc95920af829965f201 Upstream-PR: #5723 Knowledge-Delta: MiniMax-H3 ROCm deployment and evidence boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +- .../vllm-omni/models/minimax-h3/_index.md | 7 +- .../models/minimax-h3/rules-deployment.md | 100 ++++++++++++++++++ .../vllm-omni/models/minimax-h3/rules.md | 72 +------------ 12 files changed, 120 insertions(+), 81 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ed38cfee6..df3aa58ff 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ebe93f5d1e794202598e652a45fa74a29a7f698b + previous_audited_sha: 592637015ffccec514401b700548768232d9590c audited_ref: main - audited_sha: 592637015ffccec514401b700548768232d9590c + audited_sha: f01e4bdddcb90193119f4fc95920af829965f201 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b72d9fe15..b02b57816 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 592637015ffccec514401b700548768232d9590c`. The canonical machine baseline + f01e4bdddcb90193119f4fc95920af829965f201`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index beabec34f..c5ac89cfc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 59263701` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f01e4bdd` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index acdd26d36..c151f2b47 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 59263701` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f01e4bdd` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ccd949bce..3b2b9a7b5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 59263701` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f01e4bdd` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index beb5d348d..5e512a327 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 59263701` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f01e4bdd` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index baf46e245..b5272b65f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 59263701` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f01e4bdd` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 83f150d1a..5796b6b88 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 59263701` 验证存在 +- 源码校验：以上路径均已在 `main @ f01e4bdd` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 306c08fef..1691d0e7b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 59263701`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f01e4bdd`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 59263701 \ + --to f01e4bdd \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index f1b94a9c5..526442678 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5896", "PR #5914", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5896", "PR #5914", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -61,6 +61,8 @@ confidence: high - Ascend NPU 可选择 RainFusion 稀疏 video tail，并从 BF16 checkpoint 做 online INT8；两者只在 exact T2VA/Ulysses/no-AllGather DLO 配置有完成证据，几何 fallback、TP width 与组合边界见 MMH3-1a/1d。 +- ROCm BF16 的 gfx942/MI300X 四卡与 gfx950/MI350 单卡 functional evidence、AITER backend gate、 + mutable image 和 support-table 不一致边界见 MMH3-3d；不得从 gfx architecture 名直接扩展 SKU。 - datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；packed suffix trim、SAGE 短序列 role、 mask/metadata fail-closed、AllGather-KV 禁用及性能证据边界见 MMH3-1e。 - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention @@ -91,4 +93,5 @@ PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、t H3 input matrix/media ingress，以及 text-encoder completeness、online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 [MiniMax H3 rules](rules.md#direct-代码快速入口)；checkpoint transform、quantized loader 与 -text-encoder fused-source 完整性正文见 [loading rules](rules-loading.md)。 +text-encoder fused-source 完整性正文见 [loading rules](rules-loading.md)；DLO、consumer/H100/ROCm +部署和硬件证据正文见 [deployment rules](rules-deployment.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md new file mode 100644 index 000000000..9d8644cbd --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -0,0 +1,100 @@ +--- +title: "MiniMax H3 部署与证据规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5896", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +confidence: high +--- + +# MiniMax H3 部署与证据规则 + +只有 MMH3-数字字母 是可审计规则 ID。本页承载 deployment、capacity 与 hardware +measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rules.md#direct-代码快速入口)。 + +## MMH3-3a — H3 DLO 必须保持 loader layout 与 component stage 配对 + +- 触发：H3 `OffloadPlan`、`dlo_no_use_allgather`、resident layer、text encoder/VAE staging + 或 DLO 与 CPU/layerwise offload 组合。 +- 强制：H3 先走 regular loader 形成 TP-local grouped-QKV/fused-MLP layout，再以 no-AllGather + H2D streaming 使用它；`_supports_mmap_loading=False` 是安全门，不能绕过。plan 将 + `token_refiner.blocks` 独立 stream，text encoder 的 vision/text block rank-local stream， + video/audio VAE 按 encode/decode stage on-demand，leading `transformer` blocks 只在 denoise + stage resident 并在 decode 前释放。`dlo_resident_layers>0` 必须与 no-AllGather 配对。 +- 禁止：让 encoder TP shard 进入 DiT AllGather；把 DLO+CPU offload 的早期分支优先级交给 + generic CPU-offload hook；只因 component 有 `offload_to_cpu` 就推断每个 caller 都正确 staging。 + 目标 pin 的 `_encode_audio_conditions(standalone_audios)` 没有进入 `_component_on_device`， + 会在前一个 embedded-audio context 已关闭后直接使用 CPU-bound audio VAE；没有外层 stage + context 或专属测试证明 image+standalone-audio Ref2VA 的 DLO device/performance 合同。 +- 验收：分别覆盖 text、image、video、embedded audio、standalone audio 与 decode 的 + load→use→finally-offload 次序，异常也释放；regular-loader transform sentinel 到 TP-local stream + 后仍正确；resident layers 只围住 denoise。补齐上述 standalone-audio gap 前，不能用其他 + component staging 单测宣称所有 Ref2VA 输入已覆盖。^[PR #5764] + +## MMH3-3b — consumer-GPU profile 是有边界的容量证据 + +- 触发：引用 H3 2×RTX 5090/4090 可运行性、延迟、峰值或 resident-layer 默认值。 +- 强制：5090 证据只绑定 vLLM-Omni `ae6577ea` 的一次 2×32 GiB、T2VA、1344×768、 + 124 frames、50 steps run：8m38s，约 22.6 GiB/GPU 是 sampled `nvidia-smi`，不是 allocator + high-water。4090 的 2×24 GiB/1024×576/12 resident layers 只来自 2×B300 的 5-step + capacity proxy；它不是 RTX 4090 latency 或 full-run validation。 +- 禁止：把单次未 warmed 的 5090 run 写成 benchmark，把 B300 proxy 写成 target hardware， + 或从 HBM fit 推断 host fit。每个 FL2VA/Ref2VA partition 约 135 GiB，no-AllGather worker 的 + pinned CPU master 不因 resident layers 增加而消失；recipe 要求至少 200 GiB available RAM。 +- 验收：目标硬件用单一 partition、exact TP2/no-AllGather/VAE PP2/cuDNN/eager topology 复测， + 分别记录 allocator peak、重复 latency 和 joint video/audio quality。目标 pin 引用的 + `examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh` 实际不存在，因此不能把其 + four-task/MP4 validation 描述成可执行入口；补文件或改文档后再验收。^[PR #5764] + +## MMH3-3c — H100 DFX fixture 只证明 exact nightly workload 与 payload path + +- 触发：引用 H3 4×H100 latency、throughput、peak memory、DLO 节省，或修改 X2V perf lane、 + random video reference 与 model-path resolution。 +- 强制：H100 测量只绑定 PR 中较早的 partition-path commit `e8c343d5`：4×H100 80 GiB、 + USP4+HSDP4+encoder TP4+VAE PP4 tile、FLASH_ATTN、1344×768、209 frames、24 fps、8 steps、 + seed 42、1 warmup、3 measured prompts、concurrency 1。后续 `4767c524` 才将 final cases 改成 + repo-root modular model + `--task-type`，PR 没展示 switch 后复测；final nightly 只是预期采集。 + target config 中 T2V/TI2V/V2V/DLO baseline 分别是 latency mean + 38.3252/38.1385/105.7508/38.2089 s，peak-memory mean 54219.6/54622.8/55180.8/36218.4 MB； + 它们是 result metadata，当前 completed-count gate 不消费这些阈值，不能称为 regression guard。 + predecessor partition-path 实测值为 42.5836/42.3761/117.5009/42.4543 s、 + 60244/60692/61312/40242.67 MB、 + 0.0235/0.0236/0.0085/0.0236 qps，且 profiler 开启；不得外推为 no-profiler/production 数字。 + config 对三类 metric 都存 0.9×实测；对越小越好的 latency/memory，这不是“允许 +10%”的 + 正确 upper bound，即使未来恢复断言也必须先修 directionality/threshold。 +- 强制：Ref2VA synthetic clip 先用 OpenCV 生成，再尽力用 ffmpeg/libx264 转 H.264；H3 request + 必须走 `video_reference` data URL，使 server 持久化 `source_path` 给 ffprobe/ffmpeg consumer。 + 缺 ffmpeg 时 fallback `mp4v`，但 H3 要求 H.264/H.265，因此该 fallback 不是有效 H3 兼容路径。 +- 禁止：把上述 predecessor 数字当成 final repo-root modular route 的实测，或泛化到 50-step、 + 并发及其他 topology；PR #5836 final diff 本身没有其 body 所述 RoPE 修复，后续 PR #5896 + 才补 MindIE 3D→4D adapter。特殊 payload + 当前靠 resolved model 字符串包含大小写敏感的 `MiniMax-H3` 判断；materialized env/cache path + 若不含该 token 会退回 multipart `input_reference`，所以 lane 接入本身不证明 V2V 稳定执行。 +- 验收：nightly 必须证明四个 pytest case 均实际 collected、3/3 completed、artifact/log 上传， + 并在 materialized root、offline snapshot 与 repo-id 三种 model resolution 下断言 Ref2VA 都走 + `video_reference` 且 ffprobe 为 H.264/H.265；final diff 没有为新增 re-encode/model-routing helper + 增加单测，PR 所报 77 个 video-server tests 只是既有 server/API suite。性能 gate 若启用需另 + 定义 metric directionality。 + ^[PR #5836] + +## MMH3-3d — ROCm support 必须按 SKU、镜像、拓扑和测量协议限界 + +- 触发：声明 H3 AMD/ROCm 支持，引用 gfx942/gfx950 latency，或推荐 `FLASH_ATTN`、ROCm image、 + 单卡 CPU offload/四卡并行配置。 +- 强制：`FLASH_ATTN` 只在 ROCm platform 检测 gfx942/gfx950 且 AITER 可用时解析到 AITER packed + varlen，否则回退 SDPA。gfx942 证据绑定 4×MI300X、BF16、USP4、text-encoder TP4、VAE PP4 + tile、1344×768/209 frames/50 steps：一次 excluded warmup 后三次均值，T2VA encode/denoise/ + decode/client E2E 为 0.09/244.04/4.15/267.42 s，FL2VA 为 13.98/257.58/4.11/287.07 s；输出 + 仅验证 H.264 24 FPS + 32 kHz stereo AAC。^[PR #5723] +- 强制：gfx950 只绑定单张 MI350、vLLM `0.26.0+rocm723`/HIP 7.2、BF16、CPU offload、 + 832×480、约 4 s/40 steps 的 functional run；约 0.73 s/step、55 s client E2E 包含首次 lazy + compile，不是 tuned throughput。text-encoder TP 未在 gfx950 该证据中执行。 +- 禁止：从 architecture gate 外推到 MI325X/其他 MI355X SKU；从 PR body 的“single + 4 GPU”或 + “约 2× SDPA”外推，因为 final evidence table 没有 gfx950 四卡协议/结果或 A/B；也不能用可变 + `minimax-h3` tag 代替 image digest。评论中的旧 digest 缺后续 soundfile/TorchCodec/ffmpeg 状态， + 更新镜像的回复没有给新 digest。 +- 验收：support table 的 AMD cell、footnote 与 recipe 必须一致；目标 pin 中 footnote 声称已支持， + 但 H3 行 AMD cell 仍为空，修正前不能称矩阵已完整发布。新增 SKU/task/topology 逐项记录 immutable + image/commit、软件栈、warmup/repeats、输入、各阶段时间、输出/质量检查；recipe-only diff 没有 CI + 或可执行测试，外部 gfx942 数据与 gfx950 functional observation 不能冒充持续回归 gate。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 10c8163e7..250a8a119 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -29,9 +29,10 @@ confidence: high | modular checkpoint、combined/partition task、两套 DiT、shared component | `MMH3-2d` | model index discovery → startup task selection → task-specific transformer/cache lifecycle | | request `quality`、lossless/high、dynamic Cache-DiT | `MMH3-2e` | request sampling → quality policy → request Cache-DiT runtime → denoise | | force-refresh hint、once/repeat、reinstall key | `MMH3-2f` | H3 `extra_args` validation → immutable cache config → installation key/refresh context | -| DLO、TP-local、resident layers、encoder/VAE staging | `MMH3-3a` | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | -| RTX 5090/4090、24/32 GiB、consumer profile | `MMH3-3b` | recipe measurement commit/run record → exact target topology → quality/capacity validation | -| 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | `MMH3-3c` | nightly lane → perf JSON → benchmark request encoder/result artifact | +| DLO、TP-local、resident layers、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | +| RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | +| 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | +| ROCm、gfx942/gfx950、AITER、BF16、support matrix | [MMH3-3d](rules-deployment.md#mmh3-3d-rocm-support-必须按-sku镜像拓扑和测量协议限界) | support table/footnote → recipe protocol → ROCm backend gate → exact hardware evidence | ## MMH3-1b — joint video/audio quality 与 offload 兼容性必须一起验收 @@ -253,70 +254,5 @@ confidence: high disable→enable。当前覆盖为 mock/config 合同，未证明不同 topology 下 hint 对质量/命中率的效果。 ^[PR #5853] -## MMH3-3a — H3 DLO 必须保持 loader layout 与 component stage 配对 - -- 触发：H3 `OffloadPlan`、`dlo_no_use_allgather`、resident layer、text encoder/VAE staging - 或 DLO 与 CPU/layerwise offload 组合。 -- 强制：H3 先走 regular loader 形成 TP-local grouped-QKV/fused-MLP layout，再以 no-AllGather - H2D streaming 使用它；`_supports_mmap_loading=False` 是安全门，不能绕过。plan 将 - `token_refiner.blocks` 独立 stream，text encoder 的 vision/text block rank-local stream， - video/audio VAE 按 encode/decode stage on-demand，leading `transformer` blocks 只在 denoise - stage resident 并在 decode 前释放。`dlo_resident_layers>0` 必须与 no-AllGather 配对。 -- 禁止：让 encoder TP shard 进入 DiT AllGather；把 DLO+CPU offload 的早期分支优先级交给 - generic CPU-offload hook；只因 component 有 `offload_to_cpu` 就推断每个 caller 都正确 staging。 - 目标 pin 的 `_encode_audio_conditions(standalone_audios)` 没有进入 `_component_on_device`， - 会在前一个 embedded-audio context 已关闭后直接使用 CPU-bound audio VAE；没有外层 stage - context 或专属测试证明 image+standalone-audio Ref2VA 的 DLO device/performance 合同。 -- 验收：分别覆盖 text、image、video、embedded audio、standalone audio 与 decode 的 - load→use→finally-offload 次序，异常也释放；regular-loader transform sentinel 到 TP-local stream - 后仍正确；resident layers 只围住 denoise。补齐上述 standalone-audio gap 前，不能用其他 - component staging 单测宣称所有 Ref2VA 输入已覆盖。^[PR #5764] - -## MMH3-3b — consumer-GPU profile 是有边界的容量证据 - -- 触发：引用 H3 2×RTX 5090/4090 可运行性、延迟、峰值或 resident-layer 默认值。 -- 强制：5090 证据只绑定 vLLM-Omni `ae6577ea` 的一次 2×32 GiB、T2VA、1344×768、 - 124 frames、50 steps run：8m38s，约 22.6 GiB/GPU 是 sampled `nvidia-smi`，不是 allocator - high-water。4090 的 2×24 GiB/1024×576/12 resident layers 只来自 2×B300 的 5-step - capacity proxy；它不是 RTX 4090 latency 或 full-run validation。 -- 禁止：把单次未 warmed 的 5090 run 写成 benchmark，把 B300 proxy 写成 target hardware， - 或从 HBM fit 推断 host fit。每个 FL2VA/Ref2VA partition 约 135 GiB，no-AllGather worker 的 - pinned CPU master 不因 resident layers 增加而消失；recipe 要求至少 200 GiB available RAM。 -- 验收：目标硬件用单一 partition、exact TP2/no-AllGather/VAE PP2/cuDNN/eager topology 复测， - 分别记录 allocator peak、重复 latency 和 joint video/audio quality。目标 pin 引用的 - `examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh` 实际不存在，因此不能把其 - four-task/MP4 validation 描述成可执行入口；补文件或改文档后再验收。^[PR #5764] - -## MMH3-3c — H100 DFX fixture 只证明 exact nightly workload 与 payload path - -- 触发：引用 H3 4×H100 latency、throughput、peak memory、DLO 节省，或修改 X2V perf lane、 - random video reference 与 model-path resolution。 -- 强制：H100 测量只绑定 PR 中较早的 partition-path commit `e8c343d5`：4×H100 80 GiB、 - USP4+HSDP4+encoder TP4+VAE PP4 tile、FLASH_ATTN、1344×768、209 frames、24 fps、8 steps、 - seed 42、1 warmup、3 measured prompts、concurrency 1。后续 `4767c524` 才将 final cases 改成 - repo-root modular model + `--task-type`，PR 没展示 switch 后复测；final nightly 只是预期采集。 - target config 中 T2V/TI2V/V2V/DLO baseline 分别是 latency mean - 38.3252/38.1385/105.7508/38.2089 s，peak-memory mean 54219.6/54622.8/55180.8/36218.4 MB； - 它们是 result metadata，当前 completed-count gate 不消费这些阈值，不能称为 regression guard。 - predecessor partition-path 实测值为 42.5836/42.3761/117.5009/42.4543 s、 - 60244/60692/61312/40242.67 MB、 - 0.0235/0.0236/0.0085/0.0236 qps，且 profiler 开启；不得外推为 no-profiler/production 数字。 - config 对三类 metric 都存 0.9×实测；对越小越好的 latency/memory，这不是“允许 +10%”的 - 正确 upper bound，即使未来恢复断言也必须先修 directionality/threshold。 -- 强制：Ref2VA synthetic clip 先用 OpenCV 生成，再尽力用 ffmpeg/libx264 转 H.264；H3 request - 必须走 `video_reference` data URL，使 server 持久化 `source_path` 给 ffprobe/ffmpeg consumer。 - 缺 ffmpeg 时 fallback `mp4v`，但 H3 要求 H.264/H.265，因此该 fallback 不是有效 H3 兼容路径。 -- 禁止：把上述 predecessor 数字当成 final repo-root modular route 的实测，或泛化到 50-step、 - 并发及其他 topology；PR #5836 final diff 本身没有其 body 所述 RoPE 修复，后续 PR #5896 - 才补 MindIE 3D→4D adapter。特殊 payload - 当前靠 resolved model 字符串包含大小写敏感的 `MiniMax-H3` 判断；materialized env/cache path - 若不含该 token 会退回 multipart `input_reference`，所以 lane 接入本身不证明 V2V 稳定执行。 -- 验收：nightly 必须证明四个 pytest case 均实际 collected、3/3 completed、artifact/log 上传， - 并在 materialized root、offline snapshot 与 repo-id 三种 model resolution 下断言 Ref2VA 都走 - `video_reference` 且 ffprobe 为 H.264/H.265；final diff 没有为新增 re-encode/model-routing helper - 增加单测，PR 所报 77 个 video-server tests 只是既有 server/API suite。性能 gate 若启用需另 - 定义 metric directionality。 - ^[PR #5836] - 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 From c19726c3da1c181626f328c150ceb6c70bc0e068 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 19:58:49 +0800 Subject: [PATCH 064/465] knowledge(vllm-omni): sync fc8946fc (PR #5869) Upstream-Commit: fc8946fcf542cf108fa8b2fdc69de737bbfd52d8 Upstream-PR: #5869 Knowledge-Delta: MiniCPM-o HiFT CUDA graph lifecycle and shared vocoder boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/cosyvoice3/_index.md | 15 ++++++++--- .../vllm-omni/models/minicpm-o-4-5/_index.md | 8 ++++-- .../vllm-omni/models/minicpm-o-4-5/rules.md | 25 ++++++++++++++++++- .../vllm-omni/models/step-audio2/_index.md | 11 +++++--- .../models/step-audio2/architecture.md | 6 ++--- 14 files changed, 63 insertions(+), 24 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index df3aa58ff..7338102a5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 592637015ffccec514401b700548768232d9590c + previous_audited_sha: f01e4bdddcb90193119f4fc95920af829965f201 audited_ref: main - audited_sha: f01e4bdddcb90193119f4fc95920af829965f201 + audited_sha: fc8946fcf542cf108fa8b2fdc69de737bbfd52d8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b02b57816..7fb15be8c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - f01e4bdddcb90193119f4fc95920af829965f201`. The canonical machine baseline + fc8946fcf542cf108fa8b2fdc69de737bbfd52d8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c5ac89cfc..42b5b056f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f01e4bdd` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fc8946fc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c151f2b47..cef25272f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f01e4bdd` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ fc8946fc` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3b2b9a7b5..522f77196 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f01e4bdd` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fc8946fc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5e512a327..79dd0b753 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f01e4bdd` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fc8946fc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b5272b65f..1c72a03dc 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f01e4bdd` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fc8946fc` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5796b6b88..1c1e7ab1c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f01e4bdd` 验证存在 +- 源码校验：以上路径均已在 `main @ fc8946fc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1691d0e7b..b62d1837a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f01e4bdd`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fc8946fc`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to f01e4bdd \ + --to fc8946fc \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md b/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md index 018ea33f6..3db3a3d52 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md @@ -1,15 +1,15 @@ --- title: "CosyVoice3（单架构双 stage,RAS 合并停止,TRT 加速）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoice3.yaml, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py] +sources: ["PR #5869", vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoice3.yaml, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py] --- # CosyVoice3 -以下事实在 `main @ 5d44868e` 复核。 +以下事实在 `main @ fc8946fc` 复核。 ## 名称与范围 @@ -38,6 +38,15 @@ sources: [vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoic - 依赖共享模块：[Diffusion 组件](../../components/diffusion/_index.md)的 Attention 层、SharedMemoryConnector（stage 间码流）、 [Config 组件](../../components/configuration/architecture.md)。 +- 树内 `code2wav_core/hifigan.py::HiFTGenerator` 也是 Step-Audio2 与 MiniCPM-o graph wrapper 的 + shared consumer：pre-iSTFT magnitude/phase 与 eager iSTFT/clamp 必须保持同一 inference 合同， + 修改时三家一起做 state-dict、音频与 graph/eager parity。^[PR #5869] +- 目标 pin 有未闭合的 causal consumer：base `_stft/_istft` 已移除每次 + `stft_window.to(device)`，但 `CausalHiFTGenerator.__init__` 绕过 base initializer，并把 window + 作为普通 tensor 而非 registered buffer；module `.to(cuda)` 不会迁移它，继承的 STFT 路径可能 + CPU/CUDA device mismatch。新增 Step-Audio2 parity 与 MiniCPM graph 测试只实例化 noncausal base； + 修复必须让 causal window 注册/迁移，并增加 CosyVoice3 CUDA causal inference 回归，同时复核 + base 的 MiniCPM、Step-Audio2 与 GLM-TTS consumers。^[PR #5869] ## 目录内容 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 38e5daf04..e7d22f055 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5638", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/] +sources: ["PR #3642", "PR #5382", "PR #5638", "PR #5869", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/] confidence: high --- @@ -31,6 +31,10 @@ Code2Wav 在所有平台使用树内 `MiniCPMO45Token2wav`。CUDA 可显式开 Campplus TensorRT；这是可关闭的局部加速，不替换 encoder/HiFT，也不会自动启用 Step-Audio2 的 token2wav。engine cache/profile 与 fallback 门禁见 MCPMO-1c。 +四份 bundled deploy 在 CUDA 上另默认开启 HiFT graph：只 capture pre-iSTFT 子图，按 connector +chunk/cache shape 预捕并限量 lazy capture，且不服从 stage `enforce_eager`；非 CUDA 回 eager。 +shape、显存、并发与部分-graph 性能证据边界见 MCPMO-1d。 + Thinker 的 Whisper/APM audio encoder 仍构造 dense `[B,1,T,T]` mask；chunk mask 已用 broadcasted query/key index 代替逐 row Python fill，但不改变 chunk/left-context/lookahead 边界，也不消除 O(T²) storage。语义与证据门禁见 MCPMO-2b。 @@ -40,7 +44,7 @@ broadcasted query/key index 代替逐 row Python fill，但不改变 chunk/left- ## 什么时候查这里 -- 审查 MiniCPM-o 4.5 registry、remote-code gate、TTS dependency、Code2Wav TensorRT、 +- 审查 MiniCPM-o 4.5 registry、remote-code gate、TTS dependency、Code2Wav TensorRT/HiFT graph、 batch/stage handoff 或 native duplex session。 - 问题位于共享 bridge/batching 时转到 [Model Executor rules](../../components/model-executor/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 22137095c..061e96037 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5638", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5638", "PR #5869", "PR #6154", "PR #6170", "PR #6318", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] confidence: high --- @@ -22,6 +22,7 @@ confidence: high | tokenizer/processor、`trust_remote_code` | MCPMO-1a | pipeline/config factory → `model_executor/models/minicpmo_4_5/` loader | | TTS extra、backend 初始化、空音频 | MCPMO-1b | `minicpmo_4_5_omni_tts.py::MiniCPMO45OmniTTSForConditionalGeneration`、`minicpmo_4_5_token2wav.py::MiniCPMO45Token2wav` | | Code2Wav TensorRT、DiT/Campplus、engine cache/profile | MCPMO-1c | `minicpmo_4_5_code2wav.py::MiniCPMO45Code2Wav` → `batched_token2wav.py::BatchedToken2Wav._estimator_step`；共享实现 `step_audio2/step_audio2_dit_trt.py` | +| HiFT CUDA Graph、chunk bucket、lazy capture、iSTFT | MCPMO-1d | `cuda_graph_wrapper.py::HiFTGraphWrapper` → `BatchedToken2Wav._hift_inference` → shared `HiFTGenerator` | | batch、`runtime_info`、stage handoff | MCPMO-3a/3b | `stage_input_processors/minicpmo_4_5_omni.py` → `minicpmo_4_5_omni.py::MiniCPMO45OmniForConditionalGeneration` → TTS/code2wav | | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | | instructions/persona/voice/mode update、prefill slots、context lock | MCPMO-4b | `experimental/fullduplex/minicpmo45/session.py` → runtime adapter/session runner | @@ -63,6 +64,28 @@ confidence: high 复用、不同 SM 不共享 plan、CFG 后 batch 越界以及 chunk/cache 越界。PR 未新增自动化测试， 因而这些仍是后续改动必须补证的验证缺口。 ^[PR #5638] +## MCPMO-1d — HiFT graph 只捕获稳定 pre-iSTFT 子图并限界 shape cache + +- 触发：`enable_hift_graph`、capture batch/chunk 配置、HiFT inference 分段或 streaming cache shape。 +- 强制：只在 parameter device 为 CUDA 时启用；非 CUDA 回 eager。capture bucket 由 codec chunk、 + left context、Flow lookahead、token→mel ratio 与 mel/source cache 推导；启动预捕 uncached/cached + shape，未知 final shape 最多 lazy capture 8 个，超限或无可容纳 batch 时回 eager。active stream + capture 中禁止嵌套 capture/replay并回 eager；静态输入先清零再复制真实 batch，输出 slice 后 clone。 +- 强制：graph 只覆盖 `_inference_pre_istft`；`torch.istft` 的 NOLA CPU-GPU sync 与 waveform clamp + 留在 eager `_finalize_decode`。HiFT window/harmonic IDs 必须是随 module device 移动的 non-persistent + buffers。开关独立于 stage `enforce_eager`；四份 bundled MiniCPM-o deploy 默认开启。 +- 禁止：无 connector 配置、非正 chunk、负 left context、source/mel cache 不整除时静默 capture； + 将未来 `initial_codec_chunk_frames` 当已预捕（当前会 lazy capture）；把 lazy graph limit 说成总显存 + 上界，因为启动 graphs、global pool、static tensors 与其他 graph owners 仍驻留。active-capture + 分支虽绕过 nested replay，却调用包含 NOLA sync 的完整 eager decode；未做真实 outer-graph 测试前， + 不能把日志中的 “fallback” 当成可捕获保证。custom `capture_batch_sizes` 也尚未校验正数/去重。 +- 验收：CUDA test 比较 cached/uncached graph 与 eager，CPU/mock 覆盖 lazy capture/limit fallback， + deploy test 固定默认开关；另测非 CUDA disable、invalid config、unsupported batch、nested capture 和 + variable final chunk。static input/output 与 lazy graph map 为无锁可变状态；并发 replay/capture 需先 + 证明被上层串行化或增加互斥与重叠调用测试。PR 的单 A800 profile 约 30→5 ms/chunk、 + 250→43 ms/request，只绑定部分 graph、 + 单 prompt/commit `8fa28d88`，无 repeats/端到端质量，不能泛化为稳定 speedup。^[PR #5869] + ## MCPMO-2a — registry 使用 4.5 config/version predicate - 触发：pipeline auto-detection 看到通用 `MiniCPMO` architecture。 diff --git a/knowledge/repos/vllm-omni/models/step-audio2/_index.md b/knowledge/repos/vllm-omni/models/step-audio2/_index.md index 54201d97c..d49873fef 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/_index.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/_index.md @@ -4,12 +4,12 @@ created: 2026-07-21 updated: 2026-09-02 type: index tags: [vllm-omni, models] -sources: ["PR #5638", vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py] +sources: ["PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py] --- # Step-Audio2 -以下事实在 `main @ f201b717` 复核。 +以下事实在 `main @ fc8946fc` 复核。 ## 名称与范围 @@ -39,8 +39,11 @@ sources: ["PR #5638", vllm_omni/model_executor/models/step_audio2/, vllm_omni/de 家族专属 reasoning parser（`vllm_omni/reasoning/step_audio_reasoning_parser.py`）; serving 双入口——chat completions **和** `/v1/audio/speech` （`serving_speech.py`）。 -- 独有外部运行时依赖：`onnxruntime`、`s3tokenizer`、`flashcosyvoice`、 - `hyperpyyaml`（token2wav CosyVoice 栈）。 +- 独有外部运行时依赖仍包括 `onnxruntime`、`s3tokenizer`、`hyperpyyaml`；project 依赖的 + `step-audio2` package 仍携带/标注 `flashcosyvoice`，但目标 runtime 已不再从它 import HiFT/mel。 +- HiFT 与 mel spectrogram 已改用树内 CosyVoice3 实现；strict state-dict 与数值测试对 + `flashcosyvoice` reference 校验架构/输出，但测试在缺 optional package 时整体 skip，且 + MiniCPM-o 的 `HiFTGraphWrapper` 没有接入 Step-Audio2。^[PR #5869] ## 目录内容 diff --git a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md index 04f6bf5d6..3f4aab473 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md @@ -4,12 +4,12 @@ created: 2026-07-21 updated: 2026-09-02 type: architecture tags: [vllm-omni, models] -sources: ["PR #5638", vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py] +sources: ["PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py] --- # Step-Audio2 架构 -事实在 `main @ f201b717` 复核;变体/入口速览见 [index](_index.md)。 +事实在 `main @ fc8946fc` 复核;变体/入口速览见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -23,7 +23,7 @@ sources: ["PR #5638", vllm_omni/model_executor/models/step_audio2/step_audio2_th 硬编码 backbone 类）;音频嵌入并入 ``（id 151690）位置。 - 专有 token2wav（`step_audio2_token2wav.py`）：CosyVoice 风格栈—— s3tokenizer（prompt wav 语音 token）+ ONNX 说话人嵌入 + hyperpyyaml 加载的 - flow-matching（10 步 ODE）+ flashcosyvoice HiFT 声码器 → 24 kHz;树内带说话人 + flow-matching（10 步 ODE）+ 树内 CosyVoice3 HiFT/mel 实现 → 24 kHz;树内带说话人 wav——`assets/default_female.wav` 是默认,`default_male.wav` 是备选。 - 共享加速实现：`step_audio2_dit_trt.py` 提供流式 DiT ONNX export/TRT stepper， `step_audio2_token2wav.py` 提供 Campplus TRT helper；在此 pin 上只有 MiniCPM-o 的 From 416a843a1e8c44d36fd98ec53cb89fd48077d3c7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 20:12:35 +0800 Subject: [PATCH 065/465] knowledge(vllm-omni): sync 45629a8e (PR #5840) Upstream-Commit: 45629a8e268f9fe9cde7e5eca0ad57f51c21d5c2 Upstream-PR: #5840 Knowledge-Delta: MiniMax-H3 TeaCache partition, threshold, and cache-policy boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/cache-acceleration.md | 18 +++++++--- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- .../vllm-omni/models/minimax-h3/_index.md | 4 ++- .../vllm-omni/models/minimax-h3/rules.md | 33 +++++++++++++++++-- 12 files changed, 57 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7338102a5..e1d051e10 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f01e4bdddcb90193119f4fc95920af829965f201 + previous_audited_sha: fc8946fcf542cf108fa8b2fdc69de737bbfd52d8 audited_ref: main - audited_sha: fc8946fcf542cf108fa8b2fdc69de737bbfd52d8 + audited_sha: 45629a8e268f9fe9cde7e5eca0ad57f51c21d5c2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7fb15be8c..823e7658d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - fc8946fcf542cf108fa8b2fdc69de737bbfd52d8`. The canonical machine baseline + 45629a8e268f9fe9cde7e5eca0ad57f51c21d5c2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 42b5b056f..1cab63dfc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fc8946fc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 45629a8e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index cef25272f..9b88d7df3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fc8946fc` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 45629a8e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md b/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md index 9e0dd3480..ce4042a33 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md +++ b/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md @@ -1,10 +1,10 @@ --- title: "缓存加速（cache_dit / TeaCache / stepcache / magcache）" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-09-02 type: guide tags: [vllm-omni, components, diffusion] -sources: [docs/design/feature/cache_dit.md, docs/design/feature/teacache.md, vllm_omni/diffusion/cache/] +sources: ["PR #5840", docs/design/feature/cache_dit.md, docs/design/feature/teacache.md, recipes/MiniMaxAI/MiniMax-H3.md, vllm_omni/diffusion/cache/, vllm_omni/diffusion/data.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, tests/diffusion/cache/test_teacache_extractors.py] --- # 缓存加速（cache_dit / TeaCache / stepcache / magcache） @@ -12,19 +12,27 @@ sources: [docs/design/feature/cache_dit.md, docs/design/feature/teacache.md, vll 官方 spec：`docs/design/feature/{cache_dit,teacache,prefix_caching}.md`；源码 `vllm_omni/diffusion/cache/`（`base.py`、`selector.py`、`cache_dit_backend.py`、 `teacache/`、`stepcache/`、`magcache/`、`prompt_embed_cache.py`—— -`main @ 5c390096` 复核）。 +`main @ 45629a8e` 复核）。 - **Cache-DiT**：DiT 加速库，跨 denoise step 缓存中间计算——相邻步的中间特征相似， 可复用缓存跳过冗余计算；支持三类缓存策略；标准架构自动支持，自定义架构需按 spec 写自定义实现（参照实现：Qwen-Image、LongCat-Image pipeline）。 - **TeaCache**：当相邻 timestep 的调制输入（归一化 + timestep conditioning 后） - L1 距离低于阈值时，复用上一步 transformer block 残差——**1.5x–2.0x 加速**、 - 质量损失极小（参照实现：Qwen-Image transformer）。 + 累积重标定 L1 距离低于阈值时复用上一步 transformer block 残差。`rel_l1_thresh=None` + 先交给 transformer-type default，未注册模型才回退 0.2；速度/质量必须逐模型校准，不能沿用 + 通用“1.5x–2.0x、损失极小”描述。custom extractor 必须镜像 native preprocess、block call、 + SP prepare/gather 与 postprocess 的完整 kwargs/layout/device/dtype 合同，并按 request 重置 state。 + 注意 public engine 的 omitted `cache_config` 仍可能在 backend 前注入 0.2；必须从入口追到最终值。 + hook state 实际驻留在 module 上，由 runner 在每次 generation 前 reset；只有 first step 明确强制 + compute，代码没有 last-step 特例，request 隔离仍依赖 refresh 与无交错执行。 - 其他后端：`stepcache/`、`magcache/`（同一 selector 框架下的策略变体）、 `prompt_embed_cache.py`（prompt 嵌入缓存）、`prefix_caching.md`（diffusion 侧 前缀缓存语义）。 - 兼容性例外：registry 的 `_NO_CACHE_ACCELERATION = {"NextStep11Pipeline", "AudioXPipeline"}`——这两条 pipeline 不支持 cache_dit/tea_cache。 +- MiniMax-H3 只校准 FL2VA：Ref2VA-only fail fast，combined 只 hook `transformer` 并告警 + `transformers_ref` 不缓存；具体阈值与证据见 + [MMH3-2g](../../models/minimax-h3/rules.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state)。 ## 相关 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 522f77196..2eaaacb66 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fc8946fc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 45629a8e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 79dd0b753..23cb0d1d8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fc8946fc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 45629a8e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1c72a03dc..d71523b98 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fc8946fc` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 45629a8e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1c1e7ab1c..2eed00851 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fc8946fc` 验证存在 +- 源码校验：以上路径均已在 `main @ 45629a8e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b62d1837a..f4360c29c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fc8946fc`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 45629a8e`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to fc8946fc \ + --to 45629a8e \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 526442678..9e3a43f6e 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5896", "PR #5914", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5896", "PR #5914", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -63,6 +63,8 @@ confidence: high MMH3-1a/1d。 - ROCm BF16 的 gfx942/MI300X 四卡与 gfx950/MI350 单卡 functional evidence、AITER backend gate、 mutable image 和 support-table 不一致边界见 MMH3-3d；不得从 gfx architecture 名直接扩展 SKU。 +- TeaCache 只 hook FL2VA DiT，Ref2VA-only 拒绝；model default、public 入口覆盖、request `quality` + 交互与单 H100 校准边界见 MMH3-2g。 - datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；packed suffix trim、SAGE 短序列 role、 mask/metadata fail-closed、AllGather-KV 禁用及性能证据边界见 MMH3-1e。 - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 250a8a119..81d5e0d50 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5853", "PR #5896", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5896", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] confidence: high --- @@ -29,6 +29,7 @@ confidence: high | modular checkpoint、combined/partition task、两套 DiT、shared component | `MMH3-2d` | model index discovery → startup task selection → task-specific transformer/cache lifecycle | | request `quality`、lossless/high、dynamic Cache-DiT | `MMH3-2e` | request sampling → quality policy → request Cache-DiT runtime → denoise | | force-refresh hint、once/repeat、reinstall key | `MMH3-2f` | H3 `extra_args` validation → immutable cache config → installation key/refresh context | +| TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | `MMH3-2g` | custom enabler → H3 extractor → module-resident hook state / per-generation reset | | DLO、TP-local、resident layers、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | @@ -226,8 +227,9 @@ confidence: high - 触发：修改公共 `quality` 值、H3 quality policy、startup cache adoption 或 denoise 前 prepare。 - 强制：公共层只接受 `None`、`lossless`、`high`，并保留 omitted 与 explicit lossless 的区别。 - H3 中 `lossless` 总是无 cache；`high` 总是选择 model-owned conservative profile，即使启动时没有 - cache backend；omitted 仅在启动配置为 Cache-DiT 时恢复 server generic profile，否则无 cache。 + 对 request-scoped Cache-DiT，`lossless` 产生空 target；`high` 总是选择 model-owned conservative + profile，即使启动时没有 cache backend；omitted 仅在启动配置为 Cache-DiT 时恢复 server generic + profile，否则不安装 Cache-DiT。 pipeline 必须在参数/任务/step 解析完成后、真实 denoise 紧邻之前 apply plan。 - 禁止：沿用 PR 早期描述，把无 startup cache 的 `high` 拒绝；让 unsupported quality 静默落入 lossless；在 request 未显式提供 quality 时由 sync/streaming serving 覆盖 model default。recipe @@ -254,5 +256,30 @@ confidence: high disable→enable。当前覆盖为 mock/config 合同，未证明不同 topology 下 hint 对质量/命中率的效果。 ^[PR #5853] +## MMH3-2g — H3 TeaCache 只绑定 FL2VA 校准与 request state + +- 触发：H3 选择 `tea_cache`、修改 extractor、polynomial coefficients、threshold 或 partition。 +- 强制：只允许 `fl2va`/`combined`，Ref2VA-only fail fast；combined 只 hook FL2VA `transformer` 并 + 告警 Ref2VA 不缓存。extractor 必须复刻 `_embed`、packed/SP block kwargs、final row selection 与 + video/audio update masks；缺/多 kwargs、空 blocks、shape/length 不闭合立即失败。runner 在每次 + generation 前 reset module-resident hook state；first step 强制计算，后续按累积距离计算或复用 + residual，目标 hook 没有 last-step 强制计算分支。 +- 强制：H3 coefficients 只由 70 prompts、seed 42–111、256×448、107 frames、50 steps 的 3360 + adjacent pairs 校准；`TeaCacheConfig` 收到 `None` 时模型默认 0.17，自定义 coefficients/正 threshold + 可覆盖。但 public `AsyncOmniEngine._normalize_cache_config` 在省略 cache config 时仍先注入 0.2， + 因此 CLI/public omitted 路径没有获得 0.17；recipe 显式 0.17 不受影响。 +- 禁止：把 FL2VA coefficients 用于 Ref2VA，或宣称 selector 已在 request 层强制互斥。目标 pin 中 + TeaCache 由 runner 持有；`quality=lossless` 只清除 Cache-DiT target、不会卸载 TeaCache， + `quality=high` 还会尝试在同一 transformer 安装 Cache-DiT。修复或拒绝该组合前，TeaCache server + 不得用 request `quality` 承诺 lossless 或双 backend 安全。0.2 单 H100 A/B 虽约 1.20×，LPIPS + 0.3162 是显著视觉差异；0.17 在同 prompt 仅约 3% wall reduction、LPIPS 0.0134，均非通用保证。 + ^[PR #5840] +- 验收：CPU extractor tests 覆盖 native parity、packed kwargs、mask/SP 与错误路径；backend tests + 覆盖 partition/default/override。模型 E2E 仅两 steps 并检查输出 shape，未断言真实 cache hit； + 还必须从 public omitted config 断言最终 H3 threshold，不能只直接构造 backend；覆盖 TeaCache × + omitted/lossless/high，断言只存在一个 hook/backend 或明确拒绝。module state 若可交错还需并发隔离 + 证明。单 H100 online/offline、 + Cache-DiT default/conservative 数据无多 prompt/repeats，不能升级为 gate。 + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 From 795f43577fc7d6eb85ceb033f681750b5e052d8c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 20:20:07 +0800 Subject: [PATCH 066/465] knowledge(vllm-omni): sync be48b079 (PR #5946) Upstream-Commit: be48b0797eb4c316113793bf126fe9aeb858a47c Upstream-PR: #5946 Knowledge-Delta: MiniMax-H3 GB10 unified-memory capacity and evidence boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../vllm-omni/models/minimax-h3/_index.md | 4 +++- .../models/minimax-h3/rules-deployment.md | 18 +++++++++++++++++- .../repos/vllm-omni/models/minimax-h3/rules.md | 1 + 12 files changed, 32 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e1d051e10..b4651f726 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fc8946fcf542cf108fa8b2fdc69de737bbfd52d8 + previous_audited_sha: 45629a8e268f9fe9cde7e5eca0ad57f51c21d5c2 audited_ref: main - audited_sha: 45629a8e268f9fe9cde7e5eca0ad57f51c21d5c2 + audited_sha: be48b0797eb4c316113793bf126fe9aeb858a47c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 823e7658d..3d1a56f46 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ - 45629a8e268f9fe9cde7e5eca0ad57f51c21d5c2`. The canonical machine baseline +be48b0797eb4c316113793bf126fe9aeb858a47c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1cab63dfc..f9d6b6ab9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 45629a8e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ be48b079` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9b88d7df3..92dc54987 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 45629a8e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ be48b079` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2eaaacb66..4986a9533 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 45629a8e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ be48b079` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 23cb0d1d8..be3485e11 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 45629a8e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ be48b079` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d71523b98..4c8a03028 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 45629a8e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ be48b079` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2eed00851..dc4cebf55 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 45629a8e` 验证存在 +- 源码校验：以上路径均已在 `main @ be48b079` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f4360c29c..2554f1711 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 45629a8e`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ be48b079`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 45629a8e \ + --to be48b079 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 9e3a43f6e..f2903b48c 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5896", "PR #5914", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5896", "PR #5914", "PR #5946", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -70,6 +70,8 @@ confidence: high - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention 和 eager execution；resident layers 只改变 HBM/transfer，不减少 pinned host master。 实现合同与 standalone-audio staging 缺口见 MMH3-3a，5090/4090 证据边界见 MMH3-3b。 +- DGX Spark/GB10 的 unified-memory profile 必须保持单分区 resident FP8，禁用 CPU/DLO offload + 并显式提高 sync timeout；容量、测量与未验证模式边界见 MMH3-3e。 ## ComfyUI 请求路由 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 9d8644cbd..ecb415c42 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5896", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5896", "PR #5946", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -98,3 +98,19 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule 但 H3 行 AMD cell 仍为空，修正前不能称矩阵已完整发布。新增 SKU/task/topology 逐项记录 immutable image/commit、软件栈、warmup/repeats、输入、各阶段时间、输出/质量检查；recipe-only diff 没有 CI 或可执行测试，外部 gfx942 数据与 gfx950 functional observation 不能冒充持续回归 gate。 + +## MMH3-3e — GB10 unified memory 容量证据不等于离散 GPU offload 合同 + +- 触发：引用 DGX Spark/GB10 可运行性、97.7 GiB peak、36 分钟 latency，或推荐 offload/FP8。 +- 强制：只启动一个约 135 GiB FL2VA/Ref2VA partition；约 121 GiB 可用 unified memory 下必须用 + online FP8（仅 DiT 约 62→31 GiB，encoder/VAE 仍 BF16），禁用 CPU/DLO offload，因为 host/device + 共用物理池，DLO 观察到启动后 exit -9。用 eager、CUDNN_ATTN，并保持 TP/USP/ring/VAE patch + parallel degree 全为 1，同时显式配置长 timeout。 +- 禁止：把 97.7 GiB allocator reserved header 当整机 pool peak（不含 context/非 PyTorch），或把 + 单次 50-step T2VA 写成 throughput。960×576、8 s、50 steps 的 text/denoise/decode/mux/E2E 约 + 0.25/2088/70/4.84/2169.4 s，部分 stage 来自另一次 10-step run；相同 10-step denoise 波动 + 397–490 s，只能绑定该机器/配置。^[PR #5946] +- 验收：probe 确认 HTTP 200、H.264 geometry/24 FPS 与 32 kHz stereo AAC，并记录 commit、allocator + 及整机内存。目标仅新增 recipe、无自动测试；正文实测是 T2VA。评论称 Ref2VA“相同”但无命令、 + 输入、峰值或 artifact，FL2VA 也无独立证据，不能升级为已验证模式。recipe 声称 sync timeout + 默认 1800 s，但 target `api_server.py` 默认是 600 s；2169 s full run 必须显式设置更大值。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 81d5e0d50..4d65f6932 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -34,6 +34,7 @@ confidence: high | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | | ROCm、gfx942/gfx950、AITER、BF16、support matrix | [MMH3-3d](rules-deployment.md#mmh3-3d-rocm-support-必须按-sku镜像拓扑和测量协议限界) | support table/footnote → recipe protocol → ROCm backend gate → exact hardware evidence | +| DGX Spark、GB10、unified memory、FP8、offload/OOM | [MMH3-3e](rules-deployment.md#mmh3-3e-gb10-unified-memory-容量证据不等于离散-gpu-offload-合同) | single-partition recipe → allocator/header evidence → output probe | ## MMH3-1b — joint video/audio quality 与 offload 兼容性必须一起验收 From e4872e4f572b51039c77ff6bdd9e2ebbb1bf1f8b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 20:26:36 +0800 Subject: [PATCH 067/465] knowledge(vllm-omni): sync 615de4e0 (PR #5863) Upstream-Commit: 615de4e041d1eeb78b3017845ab2f0600f2c9c41 Upstream-PR: #5863 Knowledge-Delta: MiniMax-H3 RTX PRO 6000 topology and measurement boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../vllm-omni/models/minimax-h3/_index.md | 4 +++- .../models/minimax-h3/rules-deployment.md | 19 ++++++++++++++++++- .../vllm-omni/models/minimax-h3/rules.md | 1 + 12 files changed, 33 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b4651f726..d257ceb3e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 45629a8e268f9fe9cde7e5eca0ad57f51c21d5c2 + previous_audited_sha: be48b0797eb4c316113793bf126fe9aeb858a47c audited_ref: main - audited_sha: be48b0797eb4c316113793bf126fe9aeb858a47c + audited_sha: 615de4e041d1eeb78b3017845ab2f0600f2c9c41 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3d1a56f46..41c8b7433 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -be48b0797eb4c316113793bf126fe9aeb858a47c`. The canonical machine baseline +615de4e041d1eeb78b3017845ab2f0600f2c9c41`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f9d6b6ab9..8254052b7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ be48b079` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 615de4e0` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 92dc54987..cb335ef10 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ be48b079` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 615de4e0` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4986a9533..c2c46734e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ be48b079` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 615de4e0` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index be3485e11..050541224 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ be48b079` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 615de4e0` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4c8a03028..13ba33bc7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ be48b079` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 615de4e0` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index dc4cebf55..4d3c593a7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ be48b079` 验证存在 +- 源码校验：以上路径均已在 `main @ 615de4e0` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2554f1711..13e2f3e78 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ be48b079`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 615de4e0`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to be48b079 \ + --to 615de4e0 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index f2903b48c..75eb44ea2 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5896", "PR #5914", "PR #5946", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5896", "PR #5914", "PR #5946", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -72,6 +72,8 @@ confidence: high 实现合同与 standalone-audio staging 缺口见 MMH3-3a，5090/4090 证据边界见 MMH3-3b。 - DGX Spark/GB10 的 unified-memory profile 必须保持单分区 resident FP8，禁用 CPU/DLO offload 并显式提高 sync timeout；容量、测量与未验证模式边界见 MMH3-3e。 +- RTX PRO 6000 的 2/4/8-GPU BF16 profile 保持 TP2 并扩大 Ulysses；单次 T2VA scaling、SM120 + backend、PCIe/NUMA 与未测 Ref2VA/替代拓扑边界见 MMH3-3f。 ## ComfyUI 请求路由 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index ecb415c42..019878cd0 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5896", "PR #5946", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5896", "PR #5946", "PR #5863", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -114,3 +114,20 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule 及整机内存。目标仅新增 recipe、无自动测试；正文实测是 T2VA。评论称 Ref2VA“相同”但无命令、 输入、峰值或 artifact，FL2VA 也无独立证据，不能升级为已验证模式。recipe 声称 sync timeout 默认 1800 s，但 target `api_server.py` 默认是 600 s；2169 s full run 必须显式设置更大值。 + +## MMH3-3f — RTX PRO 6000 scaling 只绑定单机 T2VA 协议 + +- 触发：引用 2/4/8×RTX PRO 6000 latency、96 GiB capacity、TP/Ulysses scaling 或 device order。 +- 强制：证据只绑定 YLX Y762、8×96 GiB、driver 580.105.08/CUDA 13.0、BF16/CUDNN_ATTN、 + 1344×768、5 s、50 steps、seed 1101、两次 warmup、单请求、默认 device order/no NUMA binding。 + 2/4/8 GPU 分别 TP2×USP1/2/4，E2E 284.76/172.32/90.48 s，1 Hz `nvidia-smi` peak + 77.49/66.44/61.07 GiB/GPU；这些不是 allocator high-water。^[PR #5863] +- 禁止：把 PCIe/NUMA 重排或 TP4 headroom variant 当已测；权重 residency 由 TP、activation 由 USP + 主导，但三点拟合的约 55 GiB floor 不是跨 shape/concurrency 定律。两 server 布局未测，且会把 + host RAM/storage 需求翻倍。Ref2VA latency/memory 明确未测；review 回复的成功附件无协议/峰值， + 不能把 T2VA 数字外推。recipe 只给可变 `vllm/vllm-omni:minimax-h3` tag，没有 vLLM SHA、image + digest 或 PyTorch version；复现前必须补齐，不能把 linked issue #5901 的关闭归因于 docs-only diff。 +- 验收：新 topology 记录实际 rank groups、device order、warmup、stage/E2E、per-rank peak 与 joint + output/quality；每个 topology 当前仅两次 warmup 后测一个请求，无 repeats/variance。recipe-only diff + 无自动回归，4/8 GPU scaling 和首次请求 19% 慢均为单机观察。SM120 显式 CUDNN 合理，因为 target + TRTLLM auto-default 只覆盖 compute major 10；这不构成其他 SM120 backend 的比较证据。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 4d65f6932..a6c7b9f4c 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -35,6 +35,7 @@ confidence: high | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | | ROCm、gfx942/gfx950、AITER、BF16、support matrix | [MMH3-3d](rules-deployment.md#mmh3-3d-rocm-support-必须按-sku镜像拓扑和测量协议限界) | support table/footnote → recipe protocol → ROCm backend gate → exact hardware evidence | | DGX Spark、GB10、unified memory、FP8、offload/OOM | [MMH3-3e](rules-deployment.md#mmh3-3e-gb10-unified-memory-容量证据不等于离散-gpu-offload-合同) | single-partition recipe → allocator/header evidence → output probe | +| RTX PRO 6000、TP2、Ulysses、2/4/8 GPU、PCIe | [MMH3-3f](rules-deployment.md#mmh3-3f-rtx-pro-6000-scaling-只绑定单机-t2va-协议) | topology → exact warmed T2VA measurements → memory method | ## MMH3-1b — joint video/audio quality 与 offload 兼容性必须一起验收 From 830e416c1601e8dc6424b5f8cbc8d1038634e63f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 20:40:15 +0800 Subject: [PATCH 068/465] knowledge(vllm-omni): sync 6ae7ff78 (PR #5541) Upstream-Commit: 6ae7ff78463683fc3d0fb3eb49f6a7bea2aaf4eb Upstream-PR: #5541 Knowledge-Delta: diffusion paged-KV request control plane and Hunyuan layout boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/paged-kv-control-plane.md | 77 +++++++++++++++++++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +- .../vllm-omni/models/hunyuan-image3/_index.md | 5 +- 11 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d257ceb3e..54b86c2a5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: be48b0797eb4c316113793bf126fe9aeb858a47c + previous_audited_sha: 615de4e041d1eeb78b3017845ab2f0600f2c9c41 audited_ref: main - audited_sha: 615de4e041d1eeb78b3017845ab2f0600f2c9c41 + audited_sha: 6ae7ff78463683fc3d0fb3eb49f6a7bea2aaf4eb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 41c8b7433..20dd1dbab 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -615de4e041d1eeb78b3017845ab2f0600f2c9c41`. The canonical machine baseline +6ae7ff78463683fc3d0fb3eb49f6a7bea2aaf4eb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8254052b7..225f88fa7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 615de4e0` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6ae7ff78` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index cb335ef10..b80f4e171 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 615de4e0` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6ae7ff78` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -34,6 +34,7 @@ sources: [] | local FlashAttention deterministic opt-in、dense/non-dense dispatch 与 config propagation | [attention rules](rules-attention.md) | | DLO DP collective wave、per-worker result queue、async pump、SHM ownership 与 shutdown | [multiprocess runtime rules](rules-output-lifecycle.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | +| Scheduler-managed diffusion KV 的请求控制面、Hunyuan layout 与未实现边界 | [paged KV control plane](paged-kv-control-plane.md) | | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | | 实验性 world-model session 生命周期、LRU 与内存统计边界 | [session state](session-state.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md b/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md new file mode 100644 index 000000000..2b8e4e04c --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md @@ -0,0 +1,77 @@ +--- +title: "Diffusion paged KV control plane" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion, scheduler] +sources: ["PR #5541", vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/diffusion_kv/config.py, vllm_omni/diffusion/diffusion_kv/request.py, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py, vllm_omni/diffusion/request.py, vllm_omni/diffusion/sched/base_scheduler.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/utils.py, tests/config/test_omni_config.py, tests/diffusion/diffusion_kv/test_config.py, tests/diffusion/diffusion_kv/test_request.py, tests/diffusion/models/hunyuan_image3/test_diffusion_kv_request.py, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/test_diffusion_config_propagation.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_model_runner.py, tests/diffusion/test_diffusion_scheduler.py] +--- + +# Diffusion paged KV control plane + +只有 `DIFFKV-数字字母` 是可审计规则 ID。本页描述目标 pin 的**请求控制面合同**；它没有 +实现物理 KV cache、block table、slot mapping、paged attention 或跨请求命中。 + +## DIFFKV-1a — mode 必须显式选择，未实现模式 fail fast + +- 触发：增加或修改 `diffusion_kv_mode`、structured/runtime config 投影、KV receive intent 或 + legacy dense KV 字段。 +- 强制：`dense_legacy` 保持默认和原行为，只有 `paged_scheduler` 启用预处理与 Scheduler 所有权； + `omni_kv_config` 是独立 transport 配置，不是 mode 别名。`paged_scheduler` 必须在初始化拒绝 + `need_recv_cache=True`，但仅携带 `kv_sender_info` 不能当成导入意图。 +- 禁止：让预留的 `paged_worker_local` 静默退回 dense；在 paged mode 接受 request/sampling 上 + 非空 `past_key_values` 或任何 `*_past_key_values`。dense mode 仍保留旧字段兼容。 +- 验收：走完整 `VllmOmniConfig.from_pipeline_config()` 与 runtime config 路径，分别覆盖默认、 + paged、unknown、worker-local、receive=true 和 sender-metadata-only，不能只测 enum parser。 + ^[PR #5541] + +## DIFFKV-1b — 预处理恰好一次，并在 admission lock 之前完成 + +- 触发：修改 Engine admission 入口、模型 preprocessor、`prepared_layout` 或 Worker step-state 初始化。 +- 强制：streaming、`add_request`、async/sync wait 与 dummy warmup 都只调用一次 + `_prepare_request_for_admission`，再把 prepared request 送入内部 admission；模型预处理必须在 + Engine condition/RPC lock 之前完成。`prepared_layout` 随 Worker payload 进入 step state；泛型字段 + 实际是 `Any | None`，Hunyuan consumer 必须用 `get_hunyuan_prepared_layout` 校验具体类型。 +- 禁止：某入口漏做或重复 chat template；把 Scheduler-owned KV request 或物理 block ID 放进 + prepared layout。Hunyuan layout 只承载 tokenizer tokens、image geometry、RoPE/positions、masks + 与 scatter indices。 +- 验收：所有入口断言 exactly once；prepared/local A/B 覆盖 CFG=1/2、含/不含 reference image， + 比较 tokens、RoPE、mask、scatter index 和 step update。reference-image case 只证明规划等价； + Worker `prepare_encode()` 调用 helper 时仍设 `allow_cond_image=False` 并拒绝非 dummy image-edit，不能宣称 paged + image editing 已可执行。^[PR #5541] + +## DIFFKV-2a — Scheduler 持有逻辑请求，Worker 不得收到可变管理状态 + +- 触发：修改逻辑 KV length、CFG row mapping、context 表达、Scheduler admission 或 Worker payload。 +- 强制：每个 CFG row 建一个 Scheduler-owned `DiffusionKVRequest`；主序列是 + `[prefix | target | suffix]`，`prefix_len` 可复用，`target_len` 每步重写，`seq_len` 是首步完整 + allocation boundary，并允许 `prefix_len + target_len < seq_len`。Hunyuan 分别从 + `gen_timestep_scatter_index[row,-1]`、生成图+timestep/guidance token 和 `real_pos[row,-1]` + 取得三段长度；prompt/reference image 已在同一 self-attention sequence，所以 `kv_contexts=()`。 +- 禁止：把独立 cross/joint context 塞入主 token axis，或把可变 `DiffusionKVRequest` 送给 Worker； + admission 必须把 tuple 移入 `SchedulerRequestState` 并清空 Worker-bound 字段。 +- 验收：CFG=1/2、suffix、reference image 与独立 context 都验证 identity/length/owner。失败路径也要 + 原子：目标 `_make_request_state` 在 `_build_sampling_params_key` 前清空原 request，后者抛错时会 + 丢失 tuple；修复前必须保留 retry gap，修复时增加失败后重试测试或延后 destructive move。 + ^[PR #5541] + +## DIFFKV-2b — 空 hash 只允许 request-local paging，禁止发布 prefix + +- 触发：增加 block hashes、native `KVCacheManager` 调用、physical cache spec/config 或 prefix 发布。 +- 强制：Hunyuan 当前保持 `block_hashes=[]` 与 `skip_reading_prefix_cache=True`，只允许请求内 page + allocation；只有模型预处理为每个可缓存完整 block 生成 canonical hash 后，才可条件调用 + `cache_blocks(request, prefix_len)`。物理 geometry 必须以后续 Worker 已加载 attention module + 暴露的 native `KVCacheSpec` 为真源，再经 Engine 配置和 Worker 初始化。 +- 禁止：空 hash 无条件发布 prefix，或从静态 HF config 复制 TP heads、dtype、PP layer ownership、 + cache sharing 和 backend tensor layout；当前合同没有跨 request/CFG semantic hit。 +- 验收：空 hash case 分配后不发布；完整测试 hash 的 delayed commit 只发布 prefix、不发布 target； + native manager facade test 不得冒充 Scheduler allocation、BlockTable、slot mapping、paged attention + 或跨请求正确性。^[PR #5541] + +## 证据边界 + +- 目标实现只提供 logical request/control-plane contracts 与 native-manager conformance tests；没有 + cache manager、physical allocation、Scheduler→Worker KV metadata、BlockTable、slot mapping 或 + paged attention。 +- 目标 Hunyuan E2E 运行 `dense_legacy`，只证明既有 dense path；不能作为 Scheduler-managed paged + data plane、attention 或 cross-request caching 正确性的证据。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c2c46734e..1731616df 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 615de4e0` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6ae7ff78` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 050541224..f8f0fff10 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 615de4e0` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6ae7ff78` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 13ba33bc7..a1e1a3c52 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 615de4e0` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6ae7ff78` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4d3c593a7..7652f32f7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 615de4e0` 验证存在 +- 源码校验：以上路径均已在 `main @ 6ae7ff78` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 13e2f3e78..01f3079c8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 615de4e0`（2026-08-09）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6ae7ff78`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 615de4e0 \ + --to 6ae7ff78 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md index fb32809af..7c52285fa 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md @@ -1,10 +1,10 @@ --- title: "HunyuanImage3" created: 2026-07-10 -updated: 2026-07-30 +updated: 2026-09-02 type: index tags: [vllm-omni, models, hunyuan-image3] -sources: [] +sources: ["PR #5541", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py] --- # HunyuanImage3 @@ -25,6 +25,7 @@ sources: [] | 遇到什么 | 查看哪里 | |---|---| | 修改公开入口、prompt、AR→DiT、`model_extras`、shared task examples、条件图、size 或 seed | [开发快速入口和 rules](rules.md#direct-开发快速入口) | +| Scheduler-managed paged KV、prepared layout、CFG logical prefix 或空 hash 边界 | [shared paged KV control plane](../../components/diffusion/paged-kv-control-plane.md) | | 理解模型和 vLLM-Omni 代码地图 | [architecture](architecture.md) | | HF 接入常见偏差 | [HF alignment pitfalls](hf-alignment-pitfalls.md) | | 运行 HF baseline | [HF baseline runbook](hf-baseline-runbook.md) | From e2cb5890ac5432afa1544e1b62dd00d7f07072e1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 20:49:26 +0800 Subject: [PATCH 069/465] knowledge(vllm-omni): sync 11f633aa (PR #5746) Upstream-Commit: 11f633aa8338ba968e0e358d4b3a517f652e33e5 Upstream-PR: #5746 Knowledge-Delta: Ming Flash TTS adapter extraction and detector invariant gap --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/components/serving/rules.md | 14 +++++++++++++- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../vllm-omni/models/ming-flash-omni/_index.md | 17 +++++++++++------ 11 files changed, 35 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 54b86c2a5..98f45ead2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 615de4e041d1eeb78b3017845ab2f0600f2c9c41 + previous_audited_sha: 6ae7ff78463683fc3d0fb3eb49f6a7bea2aaf4eb audited_ref: main - audited_sha: 6ae7ff78463683fc3d0fb3eb49f6a7bea2aaf4eb + audited_sha: 11f633aa8338ba968e0e358d4b3a517f652e33e5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 20dd1dbab..5d9226146 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6ae7ff78463683fc3d0fb3eb49f6a7bea2aaf4eb`. The canonical machine baseline +11f633aa8338ba968e0e358d4b3a517f652e33e5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 225f88fa7..6ae3076b5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6ae7ff78` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 11f633aa` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b80f4e171..325f7de8d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6ae7ff78` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 11f633aa` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1731616df..8e2c0a6fc 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6ae7ff78` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 11f633aa` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f8f0fff10..2a76d7c5f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6ae7ff78` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 11f633aa` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a1e1a3c52..81a2e0a46 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6ae7ff78` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 11f633aa` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7652f32f7..b9c5e6dfe 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6ae7ff78` 验证存在 +- 源码校验：以上路径均已在 `main @ 11f633aa` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 9d278d3c8..b66d2faec 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] +sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] confidence: high --- @@ -295,6 +295,18 @@ confidence: high AST ratchet 的 branch/legacy budget 也只能下降，但 table dispatch 是已固定的漏检形态，不能把 pre-commit 通过解释为不可绕过。`omnivoice_generator` 未被 pipeline 发出，是保留兼容事实而非 可达性证据。^[PR #5682] +- adapter 提取保持共享 dispatcher 单一：Ming Flash 的 `validate()` 原样拒绝空 input、过长/非 + 字符串 instructions、`task_type`/`language`/`x_vector_only_mode`/ + `initial_codec_chunk_frames`、`ref_audio`/`ref_text` 和非正 `max_new_tokens`；`build()` 继续委托 + server 的 `_build_ming_flash_omni_prompt()`，并返回空 `tts_params` 与固定 model type。公开校验 + 和 generation preparation 都必须经 `resolve_adapter()`，不要把同一分支加回 + `serving_speech.py`。#5746 仍保留同名、priority 50 的 legacy detector，所以当前只是**执行 + adapter 提取**：检测仍由 legacy 条目先命中，随后以相同 model-type 名解析到 adapter；清理 + detector 时必须先用 detection union/oracle 证明 `ming_tts` 对 Ming dense architecture 的优先级 + 不变。当前 target 还违反 `test_tts_detection.py::test_legacy_detectors_have_no_adapter` 的全局 + 不变量（legacy 名必须 `resolve_adapter(...) is None`）；只删除旧的 + `test_ming_flash_omni_not_migrated` 不能修复它。验收必须移除该同名 legacy 条目并让 adapter + 自身保留所需 priority，或显式重定义并成套更新 union 不变量测试。^[PR #5746] - 证据：CPU oracle/ratchet suite 支撑检测等价；L20X VoxCPM2 只证明 architecture path 到达 warmup，随后因基线同样复现的 vLLM skew 退出，未证明成功 speech endpoint/audio E2E。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 01f3079c8..d22c0be2d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6ae7ff78`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 11f633aa`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 6ae7ff78 \ + --to 11f633aa \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md b/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md index 363a843b9..d75bf5ab5 100644 --- a/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md @@ -1,15 +1,15 @@ --- title: "Ming-flash-omni（BailingMM2,4 拓扑全模态）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-08-10 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/models/ming_flash_omni/, vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py] +sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/models/ming_flash_omni/, vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py, vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py, "PR #5746"] --- # Ming-flash-omni -以下事实在 `main @ 5d44868e` 复核。 +以下事实在 `main @ 11f633aa` 复核。 ## 名称与范围 @@ -43,6 +43,7 @@ sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/ | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | 文本桥 vs hidden 桥、CFG 伴随、ByT5 字形路径 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | +| speech API 的 detection/adapter union | [Serving SERV-5e](../../components/serving/rules.md#serv-5e-tts-detection-从-adapter-metadata-的有序并集派生) | `ming_tts` 消歧、请求校验和 prompt delegation | ## 配置与 checkpoint 差异 @@ -59,7 +60,11 @@ sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/ ## 什么时候查这里 -- 审查 ming_flash_omni 任一拓扑、BailingMM2 名字路由或 serving 消歧; - serving_speech 注释明示本家族**刻意不迁**到 TTS adapter 框架——别"顺手 - 统一"。 +- 审查 ming_flash_omni 任一拓扑、BailingMM2 名字路由或 serving 消歧。talker-only speech + 已由 `MingFlashOmniTTSAdapter(stage_keys={"ming_tts"})` 负责请求校验和 prompt 构建；它继续 + 调用 server 上的 `_build_ming_flash_omni_prompt()`，是提取而非 prompt 合同改写。检测端仍有 + 同名高优先级 legacy detector 保护 `ming_tts` 与 Ming dense architecture 的消歧；不要因已注册 + adapter 就假定 legacy detector 已移除，也不要把模型分支重新塞回共享 dispatcher。此过渡态 + 与 detection suite 的“legacy detector 不得同时有 adapter”不变量冲突；后续修复应把优先级 + 转移给 adapter 并移除同名 legacy 条目，同时验证 Ming dense architecture 仍正确消歧。 - 语义验收见 [model-validation](../../review/guides/model-validation.md)。 From 3ed7f2972346757cbf8e2295f6cc37f2adf19068 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 20:59:40 +0800 Subject: [PATCH 070/465] knowledge(vllm-omni): sync 36135ef0 (PR #5311) Upstream-Commit: 36135ef09ae57fa4018f454949665c37ac42fe22 Upstream-PR: #5311 Knowledge-Delta: LingBot T2I/TI2V request, conditioning, and output contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 +- .../vllm-omni/models/lingbot-video/_index.md | 28 ++++++ .../vllm-omni/models/lingbot-video/rules.md | 92 +++++++++++++++++++ 11 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/lingbot-video/_index.md create mode 100644 knowledge/repos/vllm-omni/models/lingbot-video/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 98f45ead2..b8e0cb628 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6ae7ff78463683fc3d0fb3eb49f6a7bea2aaf4eb + previous_audited_sha: 11f633aa8338ba968e0e358d4b3a517f652e33e5 audited_ref: main - audited_sha: 11f633aa8338ba968e0e358d4b3a517f652e33e5 + audited_sha: 36135ef09ae57fa4018f454949665c37ac42fe22 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5d9226146..abb2b799f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -11f633aa8338ba968e0e358d4b3a517f652e33e5`. The canonical machine baseline +36135ef09ae57fa4018f454949665c37ac42fe22`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6ae3076b5..f09143655 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 11f633aa` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 36135ef0` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 325f7de8d..5f5b361e3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 11f633aa` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 36135ef0` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8e2c0a6fc..3e7363491 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 11f633aa` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 36135ef0` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2a76d7c5f..889e5a9c7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 11f633aa` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 36135ef0` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 81a2e0a46..c0bd0398c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 11f633aa` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 36135ef0` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b9c5e6dfe..6c28e8620 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 11f633aa` 验证存在 +- 源码校验：以上路径均已在 `main @ 36135ef0` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d22c0be2d..2a8cd2e68 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 11f633aa`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 36135ef0`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 11f633aa \ + --to 36135ef0 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/lingbot-video/_index.md b/knowledge/repos/vllm-omni/models/lingbot-video/_index.md new file mode 100644 index 000000000..95147e594 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/lingbot-video/_index.md @@ -0,0 +1,28 @@ +--- +title: "LingBot-Video" +created: 2026-08-10 +updated: 2026-08-10 +type: index +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/lingbot_video/, vllm_omni/model_extras/lingbot_video.py, vllm_omni/diffusion/registry.py, "PR #5311"] +--- + +# LingBot-Video + +- registry architecture：`LingBotVideoPipeline`；dense 1.3B 与 MoE 30B-A3B checkpoint 共用 + pipeline，transformer 的 dense/MoE 实现不因 T2I/TI2V 接入而分叉。 +- 模式：已有 T2V，加 T2I 与单图 TI2V；离线走共享 text-to-image/image-to-video prompt + builder，在线分别走 `/v1/images/generations` 与 `/v1/videos`。 +- 当前公开范围仅单 request、单 output、最多一张 reference image；Refiner、生产 serving 的 + SP/USP/HSDP/CFG parallel/VAE patch parallel、component/layerwise offload、EP/FusedMoE、 + FP8 experts 与 Cache-DiT 均未暴露或缺少已落地证据（内部 `_generate` 已有 CFG-parallel 参数， + 不能据此宣称 serving 可用）。 +- owner 边界：共享入口传播/4xx 合同先查 [Serving rules](../../components/serving/rules.md)， + 通用 diffusion lifecycle 查 [Diffusion rules](../../components/diffusion/rules.md)，本模型的 + mode、帧数、图像条件和输出 key 查 [rules](rules.md)。 + +## 什么时候查这里 + +- PR 涉及 `lingbot_video`、`LingBotVideoPipeline`、T2I/T2V/TI2V 选择、LingBot 分辨率预设、 + causal-VAE 帧网格或首帧条件。 +- 性能数字只能回到可复现 benchmark；本页不把 PR 的单机耗时/显存结果升级为 gate。 diff --git a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md new file mode 100644 index 000000000..65f6b288f --- /dev/null +++ b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md @@ -0,0 +1,92 @@ +--- +title: "LingBot-Video 规则" +created: 2026-08-10 +updated: 2026-08-10 +type: rule +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/diffusion/models/lingbot_video/image_condition.py, vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/model_extras/lingbot_video.py, vllm_omni/model_extras/registry.py, tests/diffusion/models/lingbot_video/test_request_utils.py, tests/diffusion/models/lingbot_video/test_image_condition.py, tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py, tests/entrypoints/openai_api/test_image_server.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5311", "Issue #5883"] +--- + +# LingBot-Video 规则 + +只有 `LBV-数字字母` 是可审计规则 ID。共享请求传播与错误一致性先应用 +[SERV-4c/4d](../../components/serving/rules.md#serv-4c-入口接受必须闭环到每个生产消费者)， +本页只写 LingBot 专有语义。 + +## Direct 代码快速入口 + +| 信号 | 规则 | 第一批源码 | +|---|---|---| +| modalities、caption、negative prompt、size、duration、frames | LBV-1a/1b | `request_utils.py::{resolve_lingbot_mode,normalize_lingbot_request}` | +| reference image、center crop、VLM/VAE、clean prefix、RNG | LBV-2a | `image_condition.py::{prepare_ti2v_image_condition,apply_clean_prefix}` → pipeline denoise loop | +| image/video output key、HTTP 400、n>1 | LBV-2b | `pipeline_lingbot_video.py::{forward,get_lingbot_video_post_process_func}` | + +## LBV-1a — mode 由 output modality 与单图 cardinality 唯一决定 + +- 触发：修改 modalities、prompt envelope、reference media 或 T2I/T2V/TI2V prompt builder。 +- 强制：`modalities=["image"]` 且无输入图是 T2I；`["video"]` 无图是 T2V、恰好一图是 TI2V。 + image output 不接收参考图，TI2V 不接收多图；reference video/audio 均拒绝。 + modality 缺失且无图仅作 legacy T2V fallback，并 `warning_once`；有图却缺 modality 必须拒绝， + 不能猜 I2V。model-extras 的 T2I/TI2V builder 必须写显式 modality，且 TI2V 只收一个 PIL + image。 + prompt 可是 plain string 或 mapping。mapping 的 `caption` 为标量字符串时保持原文，为结构时 + 做紧凑 JSON；没有 `caption` 时去除 runtime fields 后把其余结构序列化。空 caption、空结构或 + 不可序列化值 fail closed。 +- 禁止：依据是否带图猜 output modality；用 truthiness 合并 negative prompt；让 video/audio + reference 或多图进入 pipeline。 +- 验收：三种 mode、缺失 modality fallback/拒绝、多图与异类 media、plain/structured/empty + caption 都覆盖；`negative_prompt` 优先级为 sampling `extra_args` 显式 key > prompt envelope + 非 `None` 值 > + mode default。显式空字符串与省略不同，禁止用 truthiness 合并。 + +## LBV-1b — 一次 normalization 产出 pipeline 的完整 consumer view + +- 触发：修改 frame/fps/duration、尺寸、steps/guidance/shift/output type 或来源优先级。 +- 强制：帧数来源为 model-specific `extra_args/prompt` 的 `num_frames` > sampling 中非 sentinel 的 + `num_frames` > `seconds|duration × fps` > default。视频帧向上对齐 causal VAE 的 `4n+1`； + T2I 固定 1 帧且拒绝 duration 或非 1 显式帧数。 + `seconds` 与 `duration` 同时出现拒绝；显式 `num_frames` 胜过 duration。fps/steps 必须正整数， + guidance 非负有限数，shift 正有限数，output type 只允许 `pt|np|latent`。尺寸来源互斥： + `width+height`、`size=WIDTHxHEIGHT`、或成对 `resolution+ratio` 三选一，结果必须是 16 的倍数。 +- 禁止：在 serving 再造 dimension resolver/model-specific width/height alias；把 sampling + `num_frames=1` 当成可靠的显式单帧 video——它当前是 image API 的“省略”哨兵，因此会落到默认 + 81 帧，而 model prompt/extra 的 1 仍是显式值。 +- 验收：覆盖全部来源与冲突、`4n+1` 边界、非法数值和 output type，以及 in-tree preset map。 + 当前 preset 还有方向一致性风险：表声明 value 顺序为 `(height,width)`，但例如 `16:9` 映射 + `(832,480)`，经 resolver 返回标准 `(width,height)=(480,832)`，实际是 portrait；`9:16` 同理 + 反向。现有测试冻结了该值，不能证明 ratio 名与标准 width:height 语义一致；修复需覆盖全 + preset、direct resolver、离线 builder 和两个在线 endpoint 的最终尺寸。sampling sentinel 的 + 不对称也必须先统一 omitted 表示再修改。 + +## LBV-2a — TI2V 的 VLM 与 VAE 必须共享一次几何对齐及同一 RNG 顺序 + +- 触发：修改 TI2V preprocessing、VLM/VAE conditioning、VAE posterior、generator 或 denoise loop。 +- 强制：reference image 先做保持宽高比的 resize + center crop，得到目标 H×W；同一 aligned tensor + 同时派生 Qwen3-VL image input 与 VAE clean latent，禁止两条 conditioning path 各自 resize。 + VAE posterior 必须用请求 generator 的 `latent_dist.sample(generator)`，并发生在初始 noise + latent 生成前；这是官方 TI2V parity 的 RNG 合同，换成 `mode()` 会同时改变 clean prefix 与 + 后续噪声流。 + clean temporal prefix 在初始 latents 写入一次，并在**每个** scheduler step 后重新写回； + sequential CFG、batched CFG 与 CFG-disabled 都必须覆盖。shape 要求 `[B,C,T,H,W]` 且 + batch/channel/spatial 匹配，prefix T 不能超过总 T。 +- 禁止：换用 posterior `mode()`；只在 denoise 前写 prefix；让 VLM/VAE 分别 resize；把内部 + `_generate(cfg_parallel_group=...)` 参数当成生产 serving 已暴露 CFG parallel 的证据。 +- 验收：直接断言 center crop、两条 conditioning path 复用同一几何、RNG 消耗顺序与三种 CFG + 路径的逐 step reinjection。在线 `/v1/videos` 的共享入口目前会先把 reference image 拉伸到请求尺寸，模型内部收到后已 + 丢失原始宽高比，因而上述 center-crop 不能恢复原图几何；这是跟踪中的 serving 边界，不得把 + pipeline 单测的 aspect-preserving 结果宣称为在线端到端行为。 + +## LBV-2b — modality key 与 client error 必须跨 image/video formatter 保留 + +- 触发：修改 pipeline return、postprocess、image/video formatter、batch/output count 或异常映射。 +- 强制：pipeline 返回 `DiffusionOutput(output={"image": value})` 或 `{ "video": value }`；postprocess + 必须要求二者恰好一个、按 output type 转换而不丢 key。T2I decode 必须恰好一帧并移除 frame + 轴；latent output 保持 tensor。 + `num_outputs_per_prompt != 1` 与 normalization 的 TypeError/ValueError 在 pipeline `forward` + 统一转为 `OmniClientError`，使 image/video 路径都返回 HTTP 400；不要只在 image API 加不对称 + guard。内部 `_generate` 的 image/mode 防御检查不是公开入口校验的替代品。 +- 禁止：postprocess 丢掉 modality key；让 n>1 成为 500；以内部防御性 ValueError 代替公开 + request normalization。 +- 验收：同时覆盖 formatter 的 image/video key、T2I 单帧 shape、TI2V input cardinality、最终 + image size limit、frame-count precedence 和 n>1 的 400。PR 所报 H200 单次耗时、峰值显存与 + bitwise parity 缺少本知识树可复跑的命令/产物，不能作为性能或精度 gate。 From 957a377fb3eaa3f8adc8fef3479be295c533c3ef Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 21:04:06 +0800 Subject: [PATCH 071/465] knowledge(vllm-omni): sync e1aa6eae (PR #5959) Upstream-Commit: e1aa6eae75c460cd1893bc320546e81e66973831 Upstream-PR: #5959 Knowledge-Delta: advance pin for documentation asset refresh --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- knowledge/repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b8e0cb628..edbf163cd 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 11f633aa8338ba968e0e358d4b3a517f652e33e5 + previous_audited_sha: 36135ef09ae57fa4018f454949665c37ac42fe22 audited_ref: main - audited_sha: 36135ef09ae57fa4018f454949665c37ac42fe22 + audited_sha: e1aa6eae75c460cd1893bc320546e81e66973831 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index abb2b799f..11db3f5a9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -36135ef09ae57fa4018f454949665c37ac42fe22`. The canonical machine baseline +e1aa6eae75c460cd1893bc320546e81e66973831`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f09143655..6047865b9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 36135ef0` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e1aa6eae` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5f5b361e3..86872ff8c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 36135ef0` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e1aa6eae` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3e7363491..fad6993c7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 36135ef0` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e1aa6eae` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 889e5a9c7..2ba09a84d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 36135ef0` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e1aa6eae` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c0bd0398c..f2ded6a56 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 36135ef0` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e1aa6eae` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6c28e8620..101ea9bb9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 36135ef0` 验证存在 +- 源码校验：以上路径均已在 `main @ e1aa6eae` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2a8cd2e68..86b893eb8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 36135ef0`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e1aa6eae`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 36135ef0 \ + --to e1aa6eae \ --repo \ --mode report-only ``` From 0da21e1258181effe513806a1ec49be41dae8da3 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 21:09:04 +0800 Subject: [PATCH 072/465] knowledge(vllm-omni): sync e28bf8ff (PR #5898) Upstream-Commit: e28bf8ff8b7d47a94c3c708d02e4b373dcdd0411 Upstream-PR: #5898 Knowledge-Delta: advance pin for BAGEL shared-memory test restoration --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- knowledge/repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index edbf163cd..3755d05fe 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 36135ef09ae57fa4018f454949665c37ac42fe22 + previous_audited_sha: e1aa6eae75c460cd1893bc320546e81e66973831 audited_ref: main - audited_sha: e1aa6eae75c460cd1893bc320546e81e66973831 + audited_sha: e28bf8ff8b7d47a94c3c708d02e4b373dcdd0411 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 11db3f5a9..9f3ab0134 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e1aa6eae75c460cd1893bc320546e81e66973831`. The canonical machine baseline +e28bf8ff8b7d47a94c3c708d02e4b373dcdd0411`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6047865b9..4656c585a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e1aa6eae` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e28bf8ff` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 86872ff8c..7f25fe2a2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e1aa6eae` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e28bf8ff` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index fad6993c7..e7af8570a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e1aa6eae` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e28bf8ff` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2ba09a84d..042b9cc35 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e1aa6eae` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e28bf8ff` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f2ded6a56..61fd12a4a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e1aa6eae` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e28bf8ff` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 101ea9bb9..447ae92d8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e1aa6eae` 验证存在 +- 源码校验：以上路径均已在 `main @ e28bf8ff` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 86b893eb8..5c124b813 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e1aa6eae`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e28bf8ff`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to e1aa6eae \ + --to e28bf8ff \ --repo \ --mode report-only ``` From 9a43337570719abc37ebd979b3734d6b0bb69e2b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 21:16:17 +0800 Subject: [PATCH 073/465] knowledge(vllm-omni): sync c588208c (PR #5374) Upstream-Commit: c588208cc6132b08f5066420468e047ca581fbdc Upstream-PR: #5374 Knowledge-Delta: request-level LoRA propagation --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules.md | 25 +++++++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 4 +-- 10 files changed, 34 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3755d05fe..44c29864b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e1aa6eae75c460cd1893bc320546e81e66973831 + previous_audited_sha: e28bf8ff8b7d47a94c3c708d02e4b373dcdd0411 audited_ref: main - audited_sha: e28bf8ff8b7d47a94c3c708d02e4b373dcdd0411 + audited_sha: c588208cc6132b08f5066420468e047ca581fbdc # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9f3ab0134..135f84fa7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e28bf8ff8b7d47a94c3c708d02e4b373dcdd0411`. The canonical machine baseline +c588208cc6132b08f5066420468e047ca581fbdc`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4656c585a..750dd84e6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e28bf8ff` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c588208c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7f25fe2a2..10b5a6fe5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e28bf8ff` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c588208c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e7af8570a..d100248e9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e28bf8ff` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c588208c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 042b9cc35..ea001ddaa 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e28bf8ff` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c588208c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 61fd12a4a..511598805 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e28bf8ff` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c588208c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 447ae92d8..958e9b766 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e28bf8ff` 验证存在 +- 源码校验：以上路径均已在 `main @ c588208c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index b66d2faec..a2dc53e8a 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] +sources: ["Issue #5369", "PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] confidence: high --- @@ -26,6 +26,7 @@ confidence: high | sleep/wake、partial stage/tag、idempotency、ACK、generation admission | `engine-lifecycle`：`SERV-5a`, `SERV-5b` | `entrypoints/async_omni.py::{AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni.generate}` → `worker/base.py::{handle_sleep_task,handle_wake_task}` / `diffusion/worker/diffusion_worker.py` | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | +| request-level LoRA、AR-only/multistage、streaming input update、stage cardinality | `request-contract`：`SERV-4l` | `serving_chat.py` LoRA resolve → `AsyncOmni.generate` → `AsyncOmniEngine._build_add_request_message` → stage-0 input processor | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | | video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | @@ -42,7 +43,7 @@ confidence: high | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d`, `SERV-5d` | | `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection | `SERV-5a`, `SERV-5b`, `SERV-5c`, `SERV-5e` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6c` | -| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`, `SERV-4b`, `SERV-4c`, `SERV-4d`, `SERV-4e`, `SERV-4f`, `SERV-4g`, `SERV-4h` | +| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`, `SERV-4b`, `SERV-4c`, `SERV-4d`, `SERV-4e`, `SERV-4f`, `SERV-4g`, `SERV-4h`, `SERV-4l` | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`, `SERV-4j`, `SERV-4k` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | @@ -233,6 +234,26 @@ confidence: high - 禁止：继续堆 helper、compatibility branch 或 reviewer-specific patch。 - 验收：恢复编码前 owner、consumer、删除项和 diff 预算都有可检查记录。 +### SERV-4l — request-level LoRA 必须贯穿每种 stage-0 submission 形态 + +- 触发：修改 chat adapter 解析、`AsyncOmni.generate`、ordinary/streaming request submission、 + stage-0 input processing 或 LoRA request 类型。 +- 强制：OpenAI chat `_maybe_get_adapters()` 得到的同一个 request-level `LoRARequest` 必须传入 + `AsyncOmni.generate`，再贯穿普通 `add_request_async`、streaming 首 chunk、每个 + `add_streaming_update_async` 与无 chunk/final marker，最终由 + `AsyncOmniEngine._build_add_request_message` 传给**非 diffusion stage 0** 的 vLLM input + processor。`None` 同样原样传播，不能复用上一请求 adapter。 +- 禁止：只修非流式提交；把单个 request-level LoRA 广播成每 stage adapter；绕过既有 + `resolve_sampling_params_list` 的 stage cardinality 校验；以 adapter 已加载证明 request 已选择 + 它。该参数在 target 的 engine APIs 仍标作 `Any`，不是类型安全合同；评审明确要求收窄类型， + 后续应统一为 `LoRARequest | None` 而非继续扩散 `Any`。 +- 验收：用 identity sentinel 分别覆盖普通 prompt、streaming 有 chunk、streaming 空输入/final + marker 与 `None`；单 stage sampling 参数包成长度 1，多 stage 参数长度必须等于 stage 数。 + 断言 stage-0 input processor 收到同一对象，并另外验证 downstream stage 的 LoRA 仍由对应 + `sampling_params_list[i].lora_request` 拥有，默认参数对象不被污染。GPU 验收必须先确认 adapter + ID 已加载，再证明 deterministic token/logprob 与 base 不同；#5369 另报的 `AsyncOmni.add_lora` + control-RPC 反序列化成 list 问题不在本修复范围。^[PR #5374] + ## Engine 生命周期合同 ### SERV-5a — sleep/wake 状态必须保留 stage 和 tag 作用域 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5c124b813..5daa0d6f7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e28bf8ff`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c588208c`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to e28bf8ff \ + --to c588208c \ --repo \ --mode report-only ``` From b922fb757262ce298a5ff6ac9e9be68e4ffb20c3 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 21:25:43 +0800 Subject: [PATCH 074/465] knowledge(vllm-omni): sync 33dd9a5a (PR #5843) Upstream-Commit: 33dd9a5a3e03fdae300957ce9ccffe710d8e8c86 Upstream-PR: #5843 Knowledge-Delta: diffusion admission policy and TTS detector cleanup --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-admission-wait.md | 56 +++++++++++++++++++ .../components/diffusion/step-and-batching.md | 5 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules.md | 17 +++--- knowledge/repos/vllm-omni/models/catalog.md | 4 +- .../models/ming-flash-omni/_index.md | 13 ++--- 13 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-admission-wait.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 44c29864b..49aa2ad4d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e28bf8ff8b7d47a94c3c708d02e4b373dcdd0411 + previous_audited_sha: c588208cc6132b08f5066420468e047ca581fbdc audited_ref: main - audited_sha: c588208cc6132b08f5066420468e047ca581fbdc + audited_sha: 33dd9a5a3e03fdae300957ce9ccffe710d8e8c86 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 135f84fa7..d65a0f950 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c588208cc6132b08f5066420468e047ca581fbdc`. The canonical machine baseline +33dd9a5a3e03fdae300957ce9ccffe710d8e8c86`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 750dd84e6..5fd2769cf 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c588208c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 33dd9a5a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 10b5a6fe5..34b309d39 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c588208c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 33dd9a5a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -34,6 +34,7 @@ sources: [] | local FlashAttention deterministic opt-in、dense/non-dense dispatch 与 config propagation | [attention rules](rules-attention.md) | | DLO DP collective wave、per-worker result queue、async pump、SHM ownership 与 shutdown | [multiprocess runtime rules](rules-output-lifecycle.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | +| request-wave admission coalescing、stable window、deadline 与 finite config | [admission wait rules](rules-admission-wait.md) | | Scheduler-managed diffusion KV 的请求控制面、Hunyuan layout 与未实现边界 | [paged KV control plane](paged-kv-control-plane.md) | | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-admission-wait.md b/knowledge/repos/vllm-omni/components/diffusion/rules-admission-wait.md new file mode 100644 index 000000000..80e86afc5 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-admission-wait.md @@ -0,0 +1,56 @@ +--- +title: "Diffusion admission wait rules" +created: 2026-08-10 +updated: 2026-08-10 +type: rule +tags: [vllm-omni, components, diffusion, scheduler] +sources: [docs/mkdocs/hooks/generate_argparse.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/sched/base_scheduler.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_rpc_routing.py, tests/diffusion/test_diffusion_scheduler.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5843"] +--- + +# Diffusion admission wait rules + +只有 `DIFFADM-数字字母` 是可审计规则 ID。request/step batching 的能力矩阵先查 +[step and batching](step-and-batching.md)，本页只拥有 scheduling wave 之前的 coalescing wait。 + +## DIFFADM-1a — Scheduler 决定 wait policy，Engine 只执行条件等待 + +- 触发：修改 diffusion busy loop、scheduler interface、request-wave batching、DP concurrent + coalescing 或 admission wait 配置。 +- 强制：Engine 持有 condition 和 monotonic clock，向 scheduler 请求一次内部 + `_AdmissionWaitDecision`，循环中只观测 waiting count、刷新 `stable_since`、调用 + `should_end_admission_wait()`，并用最多 2ms 的 condition wait 响应新请求/stop。Base/step/custom + scheduler 默认 no-wait；`RequestScheduler` 才拥有 static-wave coalescing policy。 +- 禁止：在 Engine 重新编码 request scheduler 的 max-batch/stability 规则；对 step scheduler + 默认等待；用 sleep 代替 condition wait；等待期间释放后不重新读取 scheduler 状态。 +- 验收：mock scheduler 证明 Engine 遵从 policy；Base/Step 返回 no-wait；RequestScheduler 分别 + 覆盖 disabled、active、已有 running wave、serial capacity、full batch、stable window、deadline + 与 stop_event。RPC/abort 唤醒和 shutdown 仍不能被 admission wait 饿死。^[PR #5843] + +## DIFFADM-1b — RequestScheduler 的等待只为可扩大的空闲 wave + +- 触发：修改 `request_batch_max_wait_ms`、`max_num_seqs`、DLO DP concurrent 或 request queue + admission。 +- 强制：仅当 max wait > 0、`max_num_running_reqs > 1` 且没有 running request 时等待。deadline + 为 start + max wait；普通稳定窗是 `min(50ms, max_wait/5)`，DP concurrent 是 + `min(300ms, max_wait/2)`。waiting 达 max batch、非空队列在稳定窗内不再增长、或到 deadline + 任一成立即结束。 +- 禁止：serial capacity 仍付固定延迟；running wave 期间等下一批；把 DP 较长稳定窗应用到普通 + request batching；把 `max_num_seqs>1` 当成 pipeline 自动支持融合——能力 gate 仍在 Engine init， + DLO DP concurrency 是现有例外。 +- 验收：对普通/DP 两种稳定窗做精确 decision 测试，并证明每次 waiting count 增长都会重置 + `stable_since`。测试 fixture 构造真实 `RequestScheduler` 时必须显式提供 + `request_batch_max_wait_ms`，否则后台 busy loop 的 `AttributeError` 会伪装成超时。^[PR #5843] + +## DIFFADM-1c — wait duration 与内部 decision 都要防止 non-finite hot spin + +- 触发：CLI/config 接受 admission duration，或 scheduler extension 改 decision 数据结构。 +- 强制：CLI argparse 与 `OmniDiffusionConfig.__post_init__` 都只接受 finite non-negative float； + `nan`、正负 `inf` 和负数在启动边界拒绝。内部 decision 保持私有，缩小第三方构造无效状态的 + 接口面。 +- 禁止：只校验 `< 0`（`nan` 会穿透并令 condition timeout 退化成 hot poll）；只在 CLI 校验而 + 放过 Python/config factory；重新公开 decision 却没有验证 finite deadline/stable window 和正 + max batch。 +- 验收：Python config 与 CLI 都覆盖 `nan|inf|-inf|negative|zero|finite positive`；docs argparse + extraction 必须为自定义 type 提供安全 stub。当前私有 `_AdmissionWaitDecision` 自身没有 + `__post_init__` invariant validation，安全性依赖 only-in-tree scheduler producer；若重新导出或 + 开放 extension，须补 deadline/stability/max-batch 非法值测试。^[PR #5843] diff --git a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md index 7d16e0697..5111559eb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md +++ b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md @@ -45,8 +45,9 @@ batching 设计页及两份用户指南已删除；迁移依赖 MkDocs redirects （`vllm_omni/diffusion/sched/`、`worker/`）。 request mode 的 `max_num_seqs>1` 要求 pipeline 显式声明 `supports_request_batch=True`，否则 -engine 初始化失败。`request_batch_max_wait_ms` 只在 request mode、支持 request batch、当前 -没有 running request 且等待值大于零时帮助突发请求成批；它用首请求延迟换 batch formation。 +engine 初始化失败（DLO DP concurrent 保留已有例外）。`request_batch_max_wait_ms` 的具体启停、 +stable window 与 deadline 已归 `RequestScheduler`；见 +[admission wait rules](rules-admission-wait.md)。它用首请求延迟换 batch formation。 ## step capability 必须验证 grouped contract diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d100248e9..c71294343 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c588208c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 33dd9a5a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ea001ddaa..db094f807 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c588208c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 33dd9a5a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 511598805..0614ed78c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c588208c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 33dd9a5a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 958e9b766..f870fb079 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c588208c` 验证存在 +- 源码校验：以上路径均已在 `main @ 33dd9a5a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index a2dc53e8a..c9db3eb24 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] +sources: ["Issue #5369", "PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] confidence: high --- @@ -307,8 +307,8 @@ confidence: high `(detect_priority, name)` 确定性排序，等优先级 detector 不得重叠。stage-key 候选、仅靠 architecture 定位的 AR entry stage、最终 model type 都从该并集派生；非集合规则覆写 `matches()`，拓扑能力覆写 `stage_serves_speech()`，不要回填 `serving_speech.py` 分支。 -- 边界：VoxCPM2 talker architecture 是高优先级权威匹配；Ming flash 的 `ming_tts` legacy - stage 先于 Ming dense 的 architecture fallback；CoVo 的通用 `fused_thinker_talker` 还需 +- 边界：VoxCPM2 talker architecture 是高优先级权威匹配；Ming flash adapter 的 `ming_tts` + stage-key 以默认 priority 100 先于 Ming dense 的 architecture fallback priority 200；CoVo 的通用 `fused_thinker_talker` 还需 architecture 确认；Audex omni 还需同部署的 `audex_code2wav`。纯 diffusion 的 `for_diffusion()` 直接绕过 adapter detection，当前 `DiffusionTTSAdapter` 没有生产 subclass。 - 验收：冻结旧 ladder 作为 oracle，覆盖 in-tree pipeline stage、adapter 声明 architecture、 @@ -321,13 +321,10 @@ confidence: high `initial_codec_chunk_frames`、`ref_audio`/`ref_text` 和非正 `max_new_tokens`；`build()` 继续委托 server 的 `_build_ming_flash_omni_prompt()`，并返回空 `tts_params` 与固定 model type。公开校验 和 generation preparation 都必须经 `resolve_adapter()`，不要把同一分支加回 - `serving_speech.py`。#5746 仍保留同名、priority 50 的 legacy detector，所以当前只是**执行 - adapter 提取**：检测仍由 legacy 条目先命中，随后以相同 model-type 名解析到 adapter；清理 - detector 时必须先用 detection union/oracle 证明 `ming_tts` 对 Ming dense architecture 的优先级 - 不变。当前 target 还违反 `test_tts_detection.py::test_legacy_detectors_have_no_adapter` 的全局 - 不变量（legacy 名必须 `resolve_adapter(...) is None`）；只删除旧的 - `test_ming_flash_omni_not_migrated` 不能修复它。验收必须移除该同名 legacy 条目并让 adapter - 自身保留所需 priority，或显式重定义并成套更新 union 不变量测试。^[PR #5746] + `serving_speech.py`。`LEGACY_TTS_DETECTORS` 当前为空，branch/legacy AST ratchet 是 27/0； + Ming Flash 只由 adapter 的 `ming_tts` stage key 检测，且 legacy 名不得同时注册 adapter。 + 验收仍须证明 `ming_tts` stage key 与 Ming dense architecture fallback 的消歧不变。 + ^[PR #5746] ^[PR #5843] - 证据：CPU oracle/ratchet suite 支撑检测等价；L20X VoxCPM2 只证明 architecture path 到达 warmup，随后因基线同样复现的 vLLM skew 退出，未证明成功 speech endpoint/audio E2E。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5daa0d6f7..babe0d989 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c588208c`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 33dd9a5a`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to c588208c \ + --to 33dd9a5a \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md b/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md index d75bf5ab5..9149804c1 100644 --- a/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md @@ -4,12 +4,12 @@ created: 2026-07-21 updated: 2026-08-10 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/models/ming_flash_omni/, vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py, vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py, "PR #5746"] +sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/models/ming_flash_omni/, vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py, vllm_omni/entrypoints/openai/tts_adapters/__init__.py, vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py, tools/pre_commit/check_tts_adapter.py, "PR #5746", "PR #5843"] --- # Ming-flash-omni -以下事实在 `main @ 11f633aa` 复核。 +以下事实在 `main @ 33dd9a5a` 复核。 ## 名称与范围 @@ -62,9 +62,8 @@ sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/ - 审查 ming_flash_omni 任一拓扑、BailingMM2 名字路由或 serving 消歧。talker-only speech 已由 `MingFlashOmniTTSAdapter(stage_keys={"ming_tts"})` 负责请求校验和 prompt 构建；它继续 - 调用 server 上的 `_build_ming_flash_omni_prompt()`，是提取而非 prompt 合同改写。检测端仍有 - 同名高优先级 legacy detector 保护 `ming_tts` 与 Ming dense architecture 的消歧；不要因已注册 - adapter 就假定 legacy detector 已移除，也不要把模型分支重新塞回共享 dispatcher。此过渡态 - 与 detection suite 的“legacy detector 不得同时有 adapter”不变量冲突；后续修复应把优先级 - 转移给 adapter 并移除同名 legacy 条目，同时验证 Ming dense architecture 仍正确消歧。 + 调用 server 上的 `_build_ming_flash_omni_prompt()`，是提取而非 prompt 合同改写。检测端的同名 + legacy detector 已移除，`LEGACY_TTS_DETECTORS=()`；`ming_tts` stage key 由 adapter 自身检测， + 其默认 priority 100 先于 Ming dense architecture fallback 的 200。不要把模型分支重新塞回共享 + dispatcher。 - 语义验收见 [model-validation](../../review/guides/model-validation.md)。 From aebd88353f297edf2cdc3cf421690f358ded8e4a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 21:30:41 +0800 Subject: [PATCH 075/465] knowledge(vllm-omni): sync 1ee2fff5 (PR #5977) Upstream-Commit: 1ee2fff532113c05d26f5fd51cb72dd3b9799eed Upstream-PR: #5977 Knowledge-Delta: Wan2.2 performance baseline reference data --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- knowledge/repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 49aa2ad4d..64b085d3e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c588208cc6132b08f5066420468e047ca581fbdc + previous_audited_sha: 33dd9a5a3e03fdae300957ce9ccffe710d8e8c86 audited_ref: main - audited_sha: 33dd9a5a3e03fdae300957ce9ccffe710d8e8c86 + audited_sha: 1ee2fff532113c05d26f5fd51cb72dd3b9799eed # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d65a0f950..897db31d4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -33dd9a5a3e03fdae300957ce9ccffe710d8e8c86`. The canonical machine baseline +1ee2fff532113c05d26f5fd51cb72dd3b9799eed`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5fd2769cf..4b121e8f5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 33dd9a5a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1ee2fff5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 34b309d39..f4ce22915 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 33dd9a5a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1ee2fff5` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c71294343..001c84425 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 33dd9a5a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1ee2fff5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index db094f807..15cda15bc 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 33dd9a5a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1ee2fff5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0614ed78c..15d570b61 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 33dd9a5a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1ee2fff5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f870fb079..0c4fe857a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 33dd9a5a` 验证存在 +- 源码校验：以上路径均已在 `main @ 1ee2fff5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index babe0d989..426cfb8b9 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 33dd9a5a`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1ee2fff5`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 33dd9a5a \ + --to 1ee2fff5 \ --repo \ --mode report-only ``` From 10bce78c42a6e5a02909486a38d30e6b28f99f8a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 21:39:51 +0800 Subject: [PATCH 076/465] knowledge(vllm-omni): sync b98b7c85 (PR #5969) Upstream-Commit: b98b7c852969f737b944120e4aaa0447b644d4e2 Upstream-PR: #5969 Knowledge-Delta: generated docs, recipe evidence, and deleted quantization sources --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/docs/_index.md | 1 + .../docs/generation-and-support-rules.md | 52 +++++++++++++++++++ knowledge/repos/vllm-omni/models/catalog.md | 4 +- .../models/hunyuan-video/architecture.md | 11 ++-- .../models/minimax-h3/rules-deployment.md | 11 ++-- .../vllm-omni/models/wan2-2/architecture.md | 11 ++-- 14 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 knowledge/repos/vllm-omni/docs/generation-and-support-rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 64b085d3e..cce60a2e1 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 33dd9a5a3e03fdae300957ce9ccffe710d8e8c86 + previous_audited_sha: 1ee2fff532113c05d26f5fd51cb72dd3b9799eed audited_ref: main - audited_sha: 1ee2fff532113c05d26f5fd51cb72dd3b9799eed + audited_sha: b98b7c852969f737b944120e4aaa0447b644d4e2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 897db31d4..359f8d97f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1ee2fff532113c05d26f5fd51cb72dd3b9799eed`. The canonical machine baseline +b98b7c852969f737b944120e4aaa0447b644d4e2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4b121e8f5..8a662b25b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1ee2fff5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b98b7c85` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f4ce22915..4c016ddae 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1ee2fff5` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b98b7c85` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 001c84425..7f8e44298 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1ee2fff5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b98b7c85` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 15cda15bc..f2c601027 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1ee2fff5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b98b7c85` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 15d570b61..04e35f9c9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1ee2fff5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b98b7c85` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0c4fe857a..8d1c6d5f6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1ee2fff5` 验证存在 +- 源码校验：以上路径均已在 `main @ b98b7c85` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/_index.md b/knowledge/repos/vllm-omni/docs/_index.md index 5e5e76ff7..a500bd9e4 100644 --- a/knowledge/repos/vllm-omni/docs/_index.md +++ b/knowledge/repos/vllm-omni/docs/_index.md @@ -23,6 +23,7 @@ sources: ["PR #5715", README.md, docs/configuration/README.md, docs/getting_star |---|---| | 判断 RFC 是否仍在进行 | [RFC status](rfcs/_index.md) | | 找上游官方设计文档与其知识树 owner | [design-doc map](design-doc-map.md) | +| generated example URL/nav、supported-model recipe 与硬件证据 | [generation and support rules](generation-and-support-rules.md) | ## 发布与安装边界 diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md new file mode 100644 index 000000000..f84190fea --- /dev/null +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -0,0 +1,52 @@ +--- +title: "Generated docs and supported-model evidence rules" +created: 2026-08-10 +updated: 2026-08-10 +type: rule +tags: [vllm-omni, docs] +sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969"] +--- + +# Generated docs and supported-model evidence rules + +只有 `DOCGEN-数字字母` 是可审计规则 ID。模型实际能力仍回 model/component owner；本页只拥有 +生成文档的 URL/navigation 与 supported-model recipe 证据边界。 + +## DOCGEN-1a — 页面 URL 与 navigation 分类是两个独立合同 + +- 触发：移动 example 分类、修改 `generate_examples.py`、MkDocs Features/Examples navigation， + 或增删 quantization example source。 +- 强制：generated page URL 始终从 source category/stem 派生；改变导航归属不能改旧 URL。 + `examples/quantization/` 的页面若存在，仍生成到 + `user_guide/examples/quantization/.md`，但 nav 只挂在 User Guide → Features → + Quantization → Generated Examples，不能同时创建 Examples → Quantization。 +- 禁止：用 nav 目标目录重写生成 URL；覆盖 hand-authored Quantization items；重复 build 累加 + Generated Examples；目标 Features/Quantization 缺失时退回错误的 Examples 分类。 +- 验收：连续运行两次仍只有一个 hook-owned group，保留 Overview 等手写项；下一次无 + quantization source 时清掉 stale generated group；目标 section 缺失时不挂载并记录 warning。 + 当前 `examples/quantization/*.py` 已全部删除，因此 target build 不应生成这些页面或链接；规则 + 保护未来重新增加 source 时的旧 URL。^[PR #5969] + +### 已知完整性缺口 + +- target 的 `.claude/skills/quantization/references/modelopt-fp8.md` 仍要求运行已删除的 + `examples/quantization/check_modelopt_fp8_export.py`。现有 + `tests/docs/test_generate_examples.py` 用构造的 `SimpleNamespace.path` 验证 nav mutation， + 不会发现这种 repository 内的悬空引用。完成该清理需要更新 skill reference，并增加覆盖真实 + 仓库路径/链接存在性的检查；在此之前不能把“删除 quantization examples”视为引用完整性已验收。 + +## DOCGEN-1b — Recipe link 与硬件 checkmark 必须共享可审计证据 + +- 触发：修改 `docs/models/supported_models.md` 的 Recipe 或 hardware 列、增加 recipe，或发布 + `recipes.vllm.ai` 页面。 +- 强制：Recipe 列是 model row → deployment recipe 的直接映射；有已确认 published page 时优先 + 链它，否则链 repository recipe，未审计 row 写 `—`。commands、参数和验证详情只由 recipe + 文件拥有，supported-model table 不复制。只有已审计 recipe 明示的硬件才能据此改该 row 的 + hardware checkmark；unlinked row 继续保留原 implementation-support metadata。 +- 禁止：把 recipe 中仅用于 platform delta/config 的提及当成运行验证；因为另一个同架构 row + 有 recipe 就传播硬件结论；把空 hardware cell 解读为已证明不支持；为避免外链而复制 recipe + 命令到 supported-model table。 +- 验收：逐个 linked row 检查 model identity、published/repository fallback、目标存在性和 recipe + 的显式硬件证据；未 linked rows 的硬件列不随本轮 audit 改动。#5969 审计的变化范围仅是 + Qwen3-Omni、Qwen-Image、Qwen-Image-2512、Qwen3-TTS CustomVoice、Higgs Audio V3 TTS 的 + NVIDIA，以及 MiniMax H3 的 NVIDIA+AMD；这不是其他 row 的平台支持 census。^[PR #5969] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 426cfb8b9..c204278b5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1ee2fff5`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b98b7c85`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -90,7 +90,7 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ --from 9235b0ae \ - --to 1ee2fff5 \ + --to b98b7c85 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-video/architecture.md b/knowledge/repos/vllm-omni/models/hunyuan-video/architecture.md index 78e0902c4..a497fb95f 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-video/architecture.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-video/architecture.md @@ -1,15 +1,15 @@ --- title: "HunyuanVideo-1.5 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-08-10 type: architecture tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py, vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py] +sources: [vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py, vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py, "PR #5969"] --- # HunyuanVideo-1.5 架构 -事实在 `main @ 5d44868e` 复核;入口/变体速览见 [index](_index.md)。 +事实在 `main @ b98b7c85` 复核;入口/变体速览见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -61,7 +61,8 @@ pin 上只有**功能面**验证入口;无精度基线或性能 gate 证据,相 - 单测：`tests/diffusion/models/hunyuan_video/test_hunyuan_video_quant_config_propagation.py`; e2e `tests/e2e/online_serving/test_hunyuan_video_15_expansion.py`;示例 `examples/online_serving/{text_to_video,image_to_video}/run_server_hunyuan_video_15.sh` - 与对应 `run_curl_hunyuan_video_15.sh` 客户端脚本;量化脚本 - `examples/quantization/quantize_hunyuanvideo_15_modelopt_fp8.py`。 + 与对应 `run_curl_hunyuan_video_15.sh` 客户端脚本。树内已无原 ModelOpt FP8 exporter，也没有 + 该模型专用的 quantization export example；审查与验证不得继续引用已删除的 example 路径。 + ^[PR #5969] - 已知未决：checkpoint 如何选 I2V 架构（推测经 model_index.json `_class_name`,解析链未追）;serving 文档是否期待用户显式传 YAML。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 019878cd0..dad642dac 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5896", "PR #5946", "PR #5863", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5863", "PR #5896", "PR #5946", "PR #5969", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -94,10 +94,11 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule “约 2× SDPA”外推，因为 final evidence table 没有 gfx950 四卡协议/结果或 A/B；也不能用可变 `minimax-h3` tag 代替 image digest。评论中的旧 digest 缺后续 soundfile/TorchCodec/ffmpeg 状态， 更新镜像的回复没有给新 digest。 -- 验收：support table 的 AMD cell、footnote 与 recipe 必须一致；目标 pin 中 footnote 声称已支持， - 但 H3 行 AMD cell 仍为空，修正前不能称矩阵已完整发布。新增 SKU/task/topology 逐项记录 immutable - image/commit、软件栈、warmup/repeats、输入、各阶段时间、输出/质量检查；recipe-only diff 没有 CI - 或可执行测试，外部 gfx942 数据与 gfx950 functional observation 不能冒充持续回归 gate。 +- 验收：support table 的 AMD cell、footnote 与 recipe 必须一致；当前 H3 行已标记 AMD 并链接 + published recipe，但这只修复 recipe-evidence 展示，不能扩大既有 SKU/task/topology 证据。 + 新增组合须逐项记录 immutable image/commit、软件栈、warmup/repeats、输入、各阶段时间、输出/质量 + 检查；recipe-only diff 没有 CI 或可执行测试，外部 gfx942 数据与 gfx950 functional observation + 不能冒充持续回归 gate。^[PR #5969] ## MMH3-3e — GB10 unified memory 容量证据不等于离散 GPU offload 合同 diff --git a/knowledge/repos/vllm-omni/models/wan2-2/architecture.md b/knowledge/repos/vllm-omni/models/wan2-2/architecture.md index ccd1596df..f719c6b5a 100644 --- a/knowledge/repos/vllm-omni/models/wan2-2/architecture.md +++ b/knowledge/repos/vllm-omni/models/wan2-2/architecture.md @@ -1,15 +1,15 @@ --- title: "Wan 2.2 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-08-10 type: architecture tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py, vllm_omni/diffusion/models/dmd2/mixin.py] +sources: [vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py, vllm_omni/diffusion/models/dmd2/mixin.py, "PR #5969"] --- # Wan 2.2 架构 -事实在 `main @ 5d44868e` 复核;变体/入口速览见 [index](_index.md);共享 +事实在 `main @ b98b7c85` 复核;变体/入口速览见 [index](_index.md);共享 diffusion 设施见 [Diffusion 组件](../../components/diffusion/_index.md)。 ## 模型专有部分与共享模块的边界 @@ -69,8 +69,9 @@ e2e（t2v 离线/在线、W4A16）、**accuracy** （perf/reliability/stability）。上列家族测试中未见 DMD2 专属测试（未做穷举 核对）。 -- 量化脚本 `examples/quantization/quantize_wan2_2_modelopt_fp8.py`;副本 - 数据并行示例 `examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml`。 +- 树内已无原 Wan2.2 ModelOpt FP8 exporter，也没有该模型专用的 quantization export example； + 审查与验证不得继续引用已删除的 example 路径。副本数据并行示例仍为 + `examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml`。^[PR #5969] - 已知未决：六架构如何按 checkpoint 选择（解析链在家族外）; `wan2_2_ti2v` key 是否用于 A14B 双专家 checkpoint 是配置惯例问题,代码不 强制。 From 4027289cf0f5b2f378276181cf3afd981f5914e7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 21:48:06 +0800 Subject: [PATCH 077/465] knowledge(vllm-omni): sync 8e44e780 (PR #5972) Upstream-Commit: 8e44e78038766b44832d6f9fd7000909517cd1cd Upstream-PR: #5972 Knowledge-Delta: MiniMax-H3 GB10 Ref2VA evidence boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/minimax-h3/_index.md | 5 +-- .../models/minimax-h3/rules-deployment.md | 32 +++++++++++++++---- 11 files changed, 41 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cce60a2e1..0ce2a0f60 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1ee2fff532113c05d26f5fd51cb72dd3b9799eed + previous_audited_sha: b98b7c852969f737b944120e4aaa0447b644d4e2 audited_ref: main - audited_sha: b98b7c852969f737b944120e4aaa0447b644d4e2 + audited_sha: 8e44e78038766b44832d6f9fd7000909517cd1cd # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 359f8d97f..564a08065 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b98b7c852969f737b944120e4aaa0447b644d4e2`. The canonical machine baseline +8e44e78038766b44832d6f9fd7000909517cd1cd`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8a662b25b..070fc185a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b98b7c85` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8e44e780` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 4c016ddae..05044a46e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b98b7c85` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8e44e780` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7f8e44298..1ba81b7ef 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b98b7c85` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8e44e780` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f2c601027..606e60e97 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b98b7c85` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8e44e780` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 04e35f9c9..5efdfadb9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b98b7c85` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8e44e780` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8d1c6d5f6..78f5ed5e5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b98b7c85` 验证存在 +- 源码校验：以上路径均已在 `main @ 8e44e780` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c204278b5..e59a22b6c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b98b7c85`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8e44e780`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 9235b0ae \ - --to b98b7c85 \ + --from b98b7c85 \ + --to 8e44e780 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 75eb44ea2..ac05444cf 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5896", "PR #5914", "PR #5946", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -71,7 +71,8 @@ confidence: high 和 eager execution；resident layers 只改变 HBM/transfer，不减少 pinned host master。 实现合同与 standalone-audio staging 缺口见 MMH3-3a，5090/4090 证据边界见 MMH3-3b。 - DGX Spark/GB10 的 unified-memory profile 必须保持单分区 resident FP8，禁用 CPU/DLO offload - 并显式提高 sync timeout；容量、测量与未验证模式边界见 MMH3-3e。 + 并显式提高 sync timeout；单图 Ref2VA/T2VA 的 exact workload、容量、热态测量与未测扩展边界见 + MMH3-3e。 - RTX PRO 6000 的 2/4/8-GPU BF16 profile 保持 TP2 并扩大 Ulysses；单次 T2VA scaling、SM120 backend、PCIe/NUMA 与未测 Ref2VA/替代拓扑边界见 MMH3-3f。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index dad642dac..968a76670 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5863", "PR #5896", "PR #5946", "PR #5969", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5863", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -102,7 +102,8 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule ## MMH3-3e — GB10 unified memory 容量证据不等于离散 GPU offload 合同 -- 触发：引用 DGX Spark/GB10 可运行性、97.7 GiB peak、36 分钟 latency，或推荐 offload/FP8。 +- 触发：引用 DGX Spark/GB10 可运行性、97.7/102.8 GiB allocator peak、T2VA/Ref2VA latency， + 或推荐 offload/FP8。 - 强制：只启动一个约 135 GiB FL2VA/Ref2VA partition；约 121 GiB 可用 unified memory 下必须用 online FP8（仅 DiT 约 62→31 GiB，encoder/VAE 仍 BF16），禁用 CPU/DLO offload，因为 host/device 共用物理池，DLO 观察到启动后 exit -9。用 eager、CUDNN_ATTN，并保持 TP/USP/ring/VAE patch @@ -111,10 +112,29 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule 单次 50-step T2VA 写成 throughput。960×576、8 s、50 steps 的 text/denoise/decode/mux/E2E 约 0.25/2088/70/4.84/2169.4 s，部分 stage 来自另一次 10-step run；相同 10-step denoise 波动 397–490 s，只能绑定该机器/配置。^[PR #5946] -- 验收：probe 确认 HTTP 200、H.264 geometry/24 FPS 与 32 kHz stereo AAC，并记录 commit、allocator - 及整机内存。目标仅新增 recipe、无自动测试；正文实测是 T2VA。评论称 Ref2VA“相同”但无命令、 - 输入、峰值或 artifact，FL2VA 也无独立证据，不能升级为已验证模式。recipe 声称 sync timeout - 默认 1800 s，但 target `api_server.py` 默认是 600 s；2169 s full run 必须显式设置更大值。 +- 强制：Ref2VA 新证据只绑定同一单机 GB10/aarch64、单 Ref2VA partition、online FP8/eager/ + CUDNN_ATTN/tiled VAE、parallel degrees 全 1、单请求、单 reference image、960×576、24 FPS、 + 8 s、flow shift 12、seed 1101。三次 10-step E2E 为 770.3–861.1 s，单次 50-step 为 + 4157.0 s；50-step denoise/per-step/decode 为 4039.6/80.79/69.4 s，allocator peak 102.8 GiB。 + 与相同 shape/steps 的 T2VA 2169.4 s 比值 1.92×只是这两个单次 observation，不是 mode 固有倍率。 + 10-step per-step 66.55–74.88 s、长请求 80.79 s 的热态差异要求容量规划用较慢端，不能承诺 + cold-run 66 s。^[PR #5972] +- 禁止：把 `X-Peak-Memory-MB: 105318.000` 当 GB10 整池峰值；它是 + `torch.cuda.max_memory_reserved`。recipe 的 1 Hz `/proc/meminfo` 交叉检查只对该 50-step + 单图请求成立：121.7 GiB 总池、`MemAvailable` 最低 7.4 GiB，推算约 114.3 GiB 已占用。 + 因此不能外推到更多图片、reference video、更大输出、并发或第二进程；这些均未测。6883-token + presentation 和 1.90× per-step 是该单图 prompt/log observation；“这些 attention tokens 导致 + denoise 变慢”没有受控对照，只是 recipe 的合理推断，不是任意 Ref2VA 输入定律。两次后续热态 + run 接近也没有温度/功耗 sensor 证据，不能据此证明稳定 thermal state。 +- 验收：probe 确认 HTTP 200、H.264 960×576/24 FPS、32 kHz stereo AAC 和约 8.032 s artifact， + 并记录 exact commit、partition、reference count/type、shape/steps、stage-header、server-log mux、 + allocator 与整机内存。FL2VA 仍无独立完成证据。PR #5972 只有 recipe diff、无可执行回归； + 其 driver 仍是 placeholder，且版本说明同时写 vLLM 0.26.0、 + vLLM-Omni `main @ e1aa6eae` 与“newer than v0.26.0”，复现时必须分别锁定两项目版本。recipe 又称 + sync timeout 默认 1800 s，但 target `api_server.py` 默认是 600 s；T2VA/Ref2VA full run 都必须 + 显式设置更大值。raw log/header、reference image、输出 artifact 与 quality metric 均未提交或链接， + 所以 HTTP/codec 描述不能替代可复核质量证据。当前文件还多出未配对的末尾 code fence，且版本 + 句含 malformed `.suggestion is v0.26.1`，发布前需修复。^[PR #5972] ## MMH3-3f — RTX PRO 6000 scaling 只绑定单机 T2VA 协议 From 62187d8e662abcfd75c9f7aed6ccdda4aee75f43 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 22:02:36 +0800 Subject: [PATCH 078/465] knowledge(vllm-omni): sync 636d44e2 (PR #5881) Upstream-Commit: 636d44e27266f794f1a440867e78f25c20cf3185 Upstream-PR: #5881 Knowledge-Delta: MUSA RMSNorm/RoPE fusion boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 16 ++++++++++--- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minimax-h3/_index.md | 6 +++-- .../vllm-omni/models/minimax-h3/rules.md | 23 +++++++++++++++---- 12 files changed, 48 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0ce2a0f60..88fb9eb9f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b98b7c852969f737b944120e4aaa0447b644d4e2 + previous_audited_sha: 8e44e78038766b44832d6f9fd7000909517cd1cd audited_ref: main - audited_sha: 8e44e78038766b44832d6f9fd7000909517cd1cd + audited_sha: 636d44e27266f794f1a440867e78f25c20cf3185 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 564a08065..799e19234 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8e44e78038766b44832d6f9fd7000909517cd1cd`. The canonical machine baseline +636d44e27266f794f1a440867e78f25c20cf3185`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 070fc185a..e9e4eea9e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8e44e780` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 636d44e2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 05044a46e..1428f07cd 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8e44e780` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 636d44e2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index c48e268e1..071ddccd5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py] confidence: high --- @@ -87,12 +87,22 @@ async output readiness、per-worker result channel 与 shutdown 合同见 tiled frequency 与 fused kernel 的 half layout，并保持未旋转 head tail 不变。若 caller 可传 无 batch 的 `[S,H,D]`，进入只接受 `[B,S,H,D]` 的 fused kernel 前必须补 batch 维，返回后仅在 本次确实补维时移除；原生 4D 输入不得被 squeeze。 +- 强制：MUSA 是独立 dispatch，不再假设兼容 CUDA。shared `RMSNorm.forward_musa` 保留 + `F.rms_norm(x, (hidden_size,), weight, eps)` 的 aten graph，供 dynamic Inductor fusion；不能改走 + 带 output mutation/`.data` 的 CUDA custom op。此共享路径直接影响 Cosmos3、HunyuanImage3、 + MiniMax-H3、Wan2.2 与 Z-Image 的 `RMSNorm` consumers。shared RoPE 只有 full-dimension、非 + interleaved NeoX layout 可走 MUSA inline 公式，当前还覆盖 Bagel/HunyuanImage3；half-dimension + 或 interleaved layout 必须退回 native。3D cos/sin 只取共享 batch 0，再插 head broadcast 维； + 这些 fallback/shape 条件是 correctness contract，不能为 fusion 删除。^[PR #5881] - 禁止：用 platform dispatch 推断所有 backend 接收同一布局；用 CPU/reference 或 mock 参数 - wiring 测试声称真实 CUDA/HIP/NPU fused kernel 已完成数值 parity。 + wiring 测试声称真实 CUDA/HIP/NPU/MUSA fused kernel 已完成数值 parity。 - 验收：native reference 覆盖 parameter dtype、FP32 accumulation、旋转维度与 untouched tail； compile 路径断言不调用 fused op，平台 mock 只作为 wiring 证据。声称 fused parity 还需目标 硬件上的同输入数值测试。mock kernel 的维度断言与 identity 回传只证明 adapter wiring/shape， - 不证明实际旋转数值。^[PR #5801] ^[PR #5896] + 不证明实际旋转数值。当前自动测试没有 MUSA dispatch/parity 或 shared-consumer census，也未证明 + `F.rms_norm` 的 FP32 accumulation；CPU H3 reference fixture 强制 `forward_native`，只覆盖 + `rot_dim=96`。硬件 A/B 是有界性能证据，不是持续回归 gate。 + ^[PR #5801] ^[PR #5881] ^[PR #5896] ## Checkpoint 与分布式加载 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1ba81b7ef..4552f0723 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8e44e780` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 636d44e2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 606e60e97..5b207f672 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8e44e780` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 636d44e2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 5efdfadb9..d9c96a716 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8e44e780` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 636d44e2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 78f5ed5e5..727d53953 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8e44e780` 验证存在 +- 源码校验：以上路径均已在 `main @ 636d44e2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e59a22b6c..2032052e5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8e44e780`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 636d44e2`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from b98b7c85 \ - --to 8e44e780 \ + --from 8e44e780 \ + --to 636d44e2 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index ac05444cf..5f2f65020 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5881", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -57,7 +57,9 @@ confidence: high 回归、XPU/ROCm 与并发边界仍见 MMH3-2c。 - H3 q/k 使用共享 RMSNorm（BF16 gamma、native FP32 accumulation）与 NeoX RoPE；每个 128 维 head 只旋转前 96 维并保留后 32 维；MindIE-SD 前还要把 H3 的 3D q/k 临时补成 - 4D batch layout，再恢复原 shape。平台布局和验证边界见 MMH3-1c。 + 4D batch layout，再恢复原 shape。MUSA dynamic compile 独立保留 aten RMSNorm graph 并内联 + full-dim NeoX RoPE，静态 rot width 从 config 派生；平台 fallback、共享 blast radius 与有界 + region-only 性能证据见 MMH3-1c/DIFF-1e。 - Ascend NPU 可选择 RainFusion 稀疏 video tail，并从 BF16 checkpoint 做 online INT8；两者只在 exact T2VA/Ulysses/no-AllGather DLO 配置有完成证据，几何 fallback、TP width 与组合边界见 MMH3-1a/1d。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index a6c7b9f4c..d0a063b2a 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5896", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5896", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] confidence: high --- @@ -58,12 +58,18 @@ confidence: high - 触发：修改 H3 q/k norm、RoPE frequency packing、head dim 或 shared fused-layer dispatch。 - 强制：H3 使用共享 `RMSNorm`，gamma 保持 BF16，而 native reduction/scale 在 FP32 累加后 - cast 回输入 dtype。CUDA/HIP eager 与 MUSA 继承的 CUDA 路径尝试 fused RMSNorm、异常时回退 - native，compile tracing 直接 native；XPU 走 native，NPU 直接调用 `npu_rms_norm` 且没有该 + cast 回输入 dtype。CUDA/HIP eager 尝试 fused RMSNorm、异常时回退 native，compile tracing + 直接 native；MUSA 独立走 `F.rms_norm` 以保留 dynamic Inductor 可融合 graph；XPU 走 native， + NPU 直接调用 `npu_rms_norm` 且没有该 fallback。RoPE 使用 NeoX `RotaryEmbedding(half_head_dim=False)`：128 维 head 只旋转前 96 维 并原样保留末 32 维；CUDA/HIP/native 把 H3 的 tiled full frequencies 转为 half kernel layout， - XPU/MUSA 明确走 native，MindIE 接收 full layout。H3 `_apply_rope` 的 q/k 是 3D `[S,H,D]`； + XPU 走 native；MUSA 对这个 full-dim、非 interleaved NeoX case 走 inline full-frequency 公式， + 其他 layout 回 native；MindIE 接收 full layout。H3 `_apply_rope` 的 q/k 是 3D `[S,H,D]`； MindIE-SD kernel 只收 4D，因此 shared adapter 必须临时变为 `[1,S,H,D]` 并把输出恢复到 3D。 +- 强制：H3 旋转宽度从 config 静态派生为 `rot_dim = 6 * rope_inv_freq_len`，默认 16 得到 96， + 不再在 forward 从 `freqs.shape[-1]` 读取动态 symbol。这样 compile 保持静态 slice，但 caller 仍 + 只调用 shared `RotaryEmbedding`、不含平台分支；config 与 frequency producer 不一致必须由 parity + test 暴露，不能静默旋转另一宽度。^[PR #5881] - 禁止：把任意 96 维随机 frequency 当成 H3 producer 合同；合法 full layout 是同一 48 维 half 的拼接。也不能从 CPU/reference 与 mocked NPU/MindIE wiring 推断真实 fused-kernel parity。 PR #5896 的新增 mock 只断言 MindIE 收到 4D、输出恢复 3D 且 identity 回传不变；没有执行真实 @@ -74,6 +80,15 @@ confidence: high `5215e03a` 且缺硬件与重复次数，只能作为有界外部观察，不能写成稳定 3.35% 保证。PR #5896 body 报告 8×Ascend 910B3、USP8/Ring1/DLO、50-step T2VA 完成并附视频，但 commit 只写 `main`、无数值对照或重复测量；它只补充运行完成证据。^[PR #5801] ^[PR #5896] +- 验收：MUSA 性能证据只绑定 1×MTT S5000、driver 3.3.5-server、immutable container digest、 + BF16 packed QKV、S=96000、7 local heads、head_dim 128/rot_dim 96、 + `torch.compile(fullgraph=True,dynamic=True)`，5 warmups、2 discarded pilots、20 个带 512 MiB + cache flush 的交错样本。该 exact Q/K RMSNorm+RoPE region 的 event median + 30.0062→1.6323 ms（18.38×），同步 wall 30.0695→1.6893 ms（17.80×）；不是 TP8、端到端 + request、其他 shape/dtype/layout 或其他 MUSA SKU 的保证。finite/shape/BF16 gate 与 direct-inline + 接近只覆盖该 A/B，且未给 correctness tolerance/reference、benchmark script 或 raw artifact； + 当前自动测试没有 MUSA dispatch/parity，CPU reference 只验证 96 维旋转和 32 维 + passthrough。^[PR #5881] ## MMH3-1d — RainFusion 只在完整、对齐的 H3 video tail 上稀疏 From 8b6df8e9b83ac14af4068474adeadb76fe002dcd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 22:09:52 +0800 Subject: [PATCH 079/465] knowledge(vllm-omni): sync ae811043 (PR #5987) Upstream-Commit: ae811043e9a3245f68e457903df77c49799b11fd Upstream-PR: #5987 Knowledge-Delta: shared task example navigation contract --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../docs/generation-and-support-rules.md | 51 ++++++++++++------- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 10 files changed, 44 insertions(+), 31 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 88fb9eb9f..8277538b3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8e44e78038766b44832d6f9fd7000909517cd1cd + previous_audited_sha: 636d44e27266f794f1a440867e78f25c20cf3185 audited_ref: main - audited_sha: 636d44e27266f794f1a440867e78f25c20cf3185 + audited_sha: ae811043e9a3245f68e457903df77c49799b11fd # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 799e19234..8c4d6b3d0 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -636d44e27266f794f1a440867e78f25c20cf3185`. The canonical machine baseline +ae811043e9a3245f68e457903df77c49799b11fd`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e9e4eea9e..a1d85c76e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 636d44e2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ae811043` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1428f07cd..daa939757 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 636d44e2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ae811043` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4552f0723..ef2e111b8 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 636d44e2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ae811043` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5b207f672..5cb79c1fe 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 636d44e2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ae811043` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d9c96a716..140705063 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 636d44e2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ae811043` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 727d53953..406100234 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 636d44e2` 验证存在 +- 源码校验：以上路径均已在 `main @ ae811043` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md index f84190fea..9fa9ae856 100644 --- a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -4,7 +4,7 @@ created: 2026-08-10 updated: 2026-08-10 type: rule tags: [vllm-omni, docs] -sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969"] +sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987"] --- # Generated docs and supported-model evidence rules @@ -12,28 +12,41 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, 只有 `DOCGEN-数字字母` 是可审计规则 ID。模型实际能力仍回 model/component owner；本页只拥有 生成文档的 URL/navigation 与 supported-model recipe 证据边界。 -## DOCGEN-1a — 页面 URL 与 navigation 分类是两个独立合同 - -- 触发：移动 example 分类、修改 `generate_examples.py`、MkDocs Features/Examples navigation， - 或增删 quantization example source。 -- 强制：generated page URL 始终从 source category/stem 派生；改变导航归属不能改旧 URL。 - `examples/quantization/` 的页面若存在，仍生成到 - `user_guide/examples/quantization/.md`，但 nav 只挂在 User Guide → Features → - Quantization → Generated Examples，不能同时创建 Examples → Quantization。 -- 禁止：用 nav 目标目录重写生成 URL；覆盖 hand-authored Quantization items；重复 build 累加 - Generated Examples；目标 Features/Quantization 缺失时退回错误的 Examples 分类。 -- 验收：连续运行两次仍只有一个 hook-owned group，保留 Overview 等手写项；下一次无 - quantization source 时清掉 stale generated group；目标 section 缺失时不挂载并记录 warning。 - 当前 `examples/quantization/*.py` 已全部删除，因此 target build 不应生成这些页面或链接；规则 - 保护未来重新增加 source 时的旧 URL。^[PR #5969] +## DOCGEN-1a — Serving sidebar 只生成共享 task 入口 + +- 触发：增删 serving example、修改 `GENERAL_EXAMPLE_SLUGS`/`is_general_example`、改变 + `generate_examples.py` 的扫描/生成顺序，或手改 User Guide → Examples navigation。 +- 强制：`offline_inference` 与 `online_serving` 只有固定 shared-task slug 可生成页面并进入 nav： + `image_to_image`、`image_to_video`、`speech_to_video`、`text_to_audio`、`text_to_image`、 + `text_to_speech`、`text_to_video`、`x_to_text`、`x_to_video_audio`。过滤必须在 page write 与 nav + update 之前执行；model-specific serving source 继续留在 `examples/`，不能重新与 task 入口混排； + 若要从文档侧发现它，必须由相应 shared task page 显式链接，不能只依赖 source directory 存在。 + 非 serving category 会全部通过 filter，仍按 category/stem 生成 + `docs/user_guide/examples//.md`。^[PR #5987] +- 强制：hook 重建 Examples 下的 generated categories，但保留不以 + `user_guide/examples/` 开头的顶层 string item（当前 `examples/README.md`）；category 和 item 均按 + 稳定顺序输出。checked-in `.nav.yml` 必须与这套 whitelist 同步，新增 shared task 要同时补 source、 + slug、shared page 内容与测试。 +- 禁止：靠 `model_display_names.yml` 把 model-specific slug 留在 sidebar；它仍只负责被构造的 + `Example` 标题校验，不覆盖 filter。不得恢复已删除的 quantization 特殊路由：target 已移除 + `EXAMPLE_NAV_TARGETS`/Features → Quantization → Generated Examples；未来非 serving + quantization source 会按通用 category 进入 Examples，除非重新设计明确合同。 +- 验收：逐个 whitelist slug 验证 offline/online 中实际存在者会生成且进入 nav，model-specific + serving slug 不生成/不入 nav且能从相应 shared page 到达，非 serving category 保持生成；再验证 + 重复 nav rebuild、preserved string 与 source-derived URL。target test 只用 synthetic + `text_to_image`/`qwen3_omni` 检查 predicate 两端，没有执行 page generation、nav serialization、 + 完整 whitelist、shared-page link coverage 或真实树 census。 + ^[PR #5987] ### 已知完整性缺口 - target 的 `.claude/skills/quantization/references/modelopt-fp8.md` 仍要求运行已删除的 - `examples/quantization/check_modelopt_fp8_export.py`。现有 - `tests/docs/test_generate_examples.py` 用构造的 `SimpleNamespace.path` 验证 nav mutation， - 不会发现这种 repository 内的悬空引用。完成该清理需要更新 skill reference，并增加覆盖真实 - 仓库路径/链接存在性的检查；在此之前不能把“删除 quantization examples”视为引用完整性已验收。 + `examples/quantization/check_modelopt_fp8_export.py`。target tests 不做 repository link census， + 不会发现这种悬空引用；完成清理需更新 skill reference 并增加真实路径存在性检查。 +- 被 filter 掉的 model-specific generated Markdown 仍作为 tracked file 留在 + `docs/user_guide/examples/`，hook 既不刷新也不删除；quickstart、design 与 feature 文档仍有直接 + 链接。因此当前链接可达不等于内容会随 source 更新。需要明确迁移链接到 shared task page，或给 + stale generated pages 定义删除/刷新策略及 link-integrity test。 ## DOCGEN-1b — Recipe link 与硬件 checkmark 必须共享可审计证据 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2032052e5..e99f637ed 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 636d44e2`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ae811043`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 8e44e780 \ - --to 636d44e2 \ + --from 636d44e2 \ + --to ae811043 \ --repo \ --mode report-only ``` From 7c393512921c77fdcf8e521f69a9f7d753d02cd5 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 22:20:51 +0800 Subject: [PATCH 080/465] knowledge(vllm-omni): sync 5e4b9f8c (PR #5958) Upstream-Commit: 5e4b9f8c6e503080f16595c578139d37759de1bb Upstream-PR: #5958 Knowledge-Delta: module ownership projection contract --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/docs/design-doc-map.md | 8 ++++- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- knowledge/repos/vllm-omni/owners/_index.md | 3 +- knowledge/repos/vllm-omni/owners/rules.md | 35 +++++++++++++++++++ 12 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/owners/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8277538b3..71e2c6a7f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 636d44e27266f794f1a440867e78f25c20cf3185 + previous_audited_sha: ae811043e9a3245f68e457903df77c49799b11fd audited_ref: main - audited_sha: ae811043e9a3245f68e457903df77c49799b11fd + audited_sha: 5e4b9f8c6e503080f16595c578139d37759de1bb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8c4d6b3d0..59891f34c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ae811043e9a3245f68e457903df77c49799b11fd`. The canonical machine baseline +5e4b9f8c6e503080f16595c578139d37759de1bb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a1d85c76e..eb9b77f91 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ae811043` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5e4b9f8c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index daa939757..7255d406e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ae811043` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5e4b9f8c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ef2e111b8..31083786f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ae811043` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5e4b9f8c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5cb79c1fe..08c8549cf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ae811043` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5e4b9f8c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 140705063..be8d56edb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ae811043` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5e4b9f8c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 406100234..6b7898b5e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ae811043` 验证存在 +- 源码校验：以上路径均已在 `main @ 5e4b9f8c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index 1d89debb1..1bb74a086 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, docs] -sources: ["PR #5139", "PR #5833", "PR #5839", "PR #5914", docs/.nav.yml, docs/design/index.md, docs/design/module/, docs/design/module/archive/README.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] +sources: ["PR #5139", "PR #5833", "PR #5839", "PR #5914", "PR #5958", .github/CODEOWNERS, docs/.nav.yml, docs/design/index.md, docs/design/module/, docs/design/module/archive/README.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] --- # 官方设计文档地图（docs/design/**） @@ -19,6 +19,12 @@ infrastructure/performance 和 21 个 module 页面编排。module frontmatter 上游刻意没有为旧 URL 留 redirect，因此外部深链会 404。owner 与 `required_reviewers` 是不同角色， 不能用 document steward/technical owner 冒充独立验证批准。 +module frontmatter 的 `owners` + `primary_code_paths` 现在是 `vllm_omni/**` ownership 的 edit-first +source map，少数 `primary_code_path_owners` 为子路径追加 owners；`.github/CODEOWNERS` 是 +last-match-wins 的 GitHub 执行投影。它没有自动生成/parity test，具体维护与冲突边界见 +[ownership projection rules](../owners/rules.md)。ownership metadata 的变化不改变下表 +runtime owner 的行为合同；仍须回 live code/tests 验证。^[PR #5958] + 此前 #5833/#5839 增加 DLO 与 quantization design；`cpu_offload_diffusion.md` 重命名为 `cpu_offload.md` 且保留 MkDocs redirect，并删除误导性的 `ray_based_execution.md`。目标 pin 仍有 `distributed/ray_utils/`、`worker_backend`、`ray_address`，但 diffusion executor 明确报 Ray backend diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e99f637ed..608552335 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ae811043`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5e4b9f8c`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 636d44e2 \ - --to ae811043 \ + --from ae811043 \ + --to 5e4b9f8c \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/owners/_index.md b/knowledge/repos/vllm-omni/owners/_index.md index aea866817..46ac09e6f 100644 --- a/knowledge/repos/vllm-omni/owners/_index.md +++ b/knowledge/repos/vllm-omni/owners/_index.md @@ -4,7 +4,7 @@ created: 2026-08-11 updated: 2026-09-02 type: index tags: [vllm-omni, models] -sources: [.github/CODEOWNERS, docs/community/governance.md] +sources: [.github/CODEOWNERS, docs/community/governance.md, docs/design/module/, "PR #5958"] --- # vLLM-Omni 负责人 @@ -29,4 +29,5 @@ sources: [.github/CODEOWNERS, docs/community/governance.md] |---|---|---| | 按 changed path 找 review owner | 上游 `.github/CODEOWNERS` | 权威且随 pin 变化；按 GitHub CODEOWNERS pattern 匹配，不从本地人员表推断 | | 确认 committer 身份与公开职责 | 上游 `docs/community/governance.md` | governance roster 是角色权威来源，不等同于每条 path 的 required reviewer | +| 维护 module frontmatter → CODEOWNERS 投影 | [ownership projection rules](rules.md) | edit-first source map、additive overlay、last-match 与双向 parity | | 关键模型与 Owner 清单 | [model_owner](model_owner.md) | 模型 → 负责人对照表 | diff --git a/knowledge/repos/vllm-omni/owners/rules.md b/knowledge/repos/vllm-omni/owners/rules.md new file mode 100644 index 000000000..22ecdf34a --- /dev/null +++ b/knowledge/repos/vllm-omni/owners/rules.md @@ -0,0 +1,35 @@ +--- +title: "Source-tree ownership projection rules" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, review] +sources: [.github/CODEOWNERS, docs/design/module/, "PR #5958"] +--- + +# Source-tree ownership projection rules + +只有 `OWN-数字字母` 是可审计规则 ID。本页维护 source-tree review routing；runtime 行为仍回对应 +component/model owner，committer 身份与职责仍回 `docs/community/governance.md`。 + +## OWN-1a — Module frontmatter 与 CODEOWNERS 必须保持可验证投影 + +- 触发：修改 `docs/design/module/*.md` 的 `owners`、`primary_code_paths` 或 + `primary_code_path_owners`，修改 `.github/CODEOWNERS` 的 `vllm_omni/**` rule，或新增跨模块 + subtree/file override。 +- 强制：先编辑 module frontmatter 作为 edit-first source map，再同步 `.github/CODEOWNERS`；GitHub + 自动 review 的执行权威仍是目标 pin 的 CODEOWNERS。`primary_code_path_owners` 是 scoped additive + overlay，命中时必须与 module base owners 取并集。CODEOWNERS 按 last-match-wins 排列 umbrella + rule 在前、submodule/file override 在后，并用最终最具体的匹配路由 changed path。module 页的 + `status: draft` 不妨碍 ownership 投影，也不把 candidate invariant 升级成 runtime 规范。^[PR #5958] +- 禁止：只改 frontmatter 或 CODEOWNERS 一侧；让 scoped overlay 替换 base owners；从 module + frontmatter 推断 docs/recipes/tests/CI/apps/out-of-tree benchmark 等直接维护的 CODEOWNERS section； + 或用 PR body/comment 的 owner 名单覆盖目标文件。没有生成器/checker 时，“已模拟若干路径”也不能 + 充当持续 gate。 +- 验收：做双向 parity——每个 `primary_code_paths`/scoped overlay 都有对应 CODEOWNERS rule，反向 + 每条声称来自 module doc 的 source-tree rule 也能回到 frontmatter；再用代表性路径模拟最终匹配， + 至少覆盖 catch-all、umbrella、subtree 和 file override。目标仓库当前没有自动 generator、parity + checker 或 regression test，因此合入前必须保留人工双向检查证据；PR prose 与目标文件冲突时按 + 目标文件路由，并把治理意图差异交给 owner 确认。 + +相关导航：[负责人索引](_index.md)、[官方设计文档地图](../docs/design-doc-map.md)。 From 33d0f9c9751f6404c24c84ca95740ed264efb6f4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 22:31:58 +0800 Subject: [PATCH 081/465] knowledge(vllm-omni): sync 740f45ab (PR #5891) Upstream-Commit: 740f45abcd2e6754d1cdded01ee9901d595ccac5 Upstream-PR: #5891 Knowledge-Delta: MiniMax-H3 NPU packed attention boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 4 +-- .../components/diffusion/rules-attention.md | 31 ++++++++++++++++++- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/minimax-h3/_index.md | 5 ++- .../models/minimax-h3/rules-deployment.md | 24 +++++++++++++- .../vllm-omni/models/minimax-h3/rules.md | 23 +++++++++++++- 13 files changed, 92 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 71e2c6a7f..6dd0dd178 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ae811043e9a3245f68e457903df77c49799b11fd + previous_audited_sha: 5e4b9f8c6e503080f16595c578139d37759de1bb audited_ref: main - audited_sha: 5e4b9f8c6e503080f16595c578139d37759de1bb + audited_sha: 740f45abcd2e6754d1cdded01ee9901d595ccac5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 59891f34c..d004c3710 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5e4b9f8c6e503080f16595c578139d37759de1bb`. The canonical machine baseline +740f45abcd2e6754d1cdded01ee9901d595ccac5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index eb9b77f91..de88f96b6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5e4b9f8c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 740f45ab` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7255d406e..439ee26a6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5e4b9f8c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 740f45ab` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -31,7 +31,7 @@ sources: [] | 理解共享职责和数据流 | [architecture](architecture.md) | | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | -| local FlashAttention deterministic opt-in、dense/non-dense dispatch 与 config propagation | [attention rules](rules-attention.md) | +| local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | | DLO DP collective wave、per-worker result queue、async pump、SHM ownership 与 shutdown | [multiprocess runtime rules](rules-output-lifecycle.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | request-wave admission coalescing、stable window、deadline 与 finite config | [admission wait rules](rules-admission-wait.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 5f40e3bf3..ed2864639 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5897", vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/config/test_omni_config.py] +sources: ["PR #5887", "PR #5891", "PR #5897", vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py] confidence: high --- @@ -24,3 +24,32 @@ confidence: high config/stage-engine field set 会分类该字段；PR 引用/运行的既有 classification test 只断言字段分类 集合与 projection dataclass 一致，没有以 non-default true 证明 structured/deploy→ `OmniDiffusionConfig`→backend 的值传播，也没有重跑 Qwen accuracy。^[PR #5887] ^[PR #5897] + +## DIFF-1g — NPU packed mask-free 是 model opt-in + backend fallback 双合同 + +- 触发：修改 `supports_packed_mask_free`、NPU `FLASH_ATTN`、packed cu/max metadata、 + `npu_attn_varlen`、`laser_input_scale` 或 MindIE-SD backend selection。 +- 强制：base backend capability 默认 false；`FlashAttentionBackend` 只在 CUDA/NPU 报 true，XPU + 仍读 mask。NPU 新分支还必须由每次 forward 的 `extra["npu_attn_varlen"]` 显式 opt in；未设置的 + Wan/Cosmos 等 caller 保持原 masked `attention_forward`。quantized KV 在 `forward_npu` 更早转入 + `forward_fa_quant_npu`，不消费这套 mask-free metadata。显式选择 NPU `FLASH_ATTN` 且 MindIE-SD + 存在时，platform 必须在首次 custom-op registry snapshot 前 eager import 它。^[PR #5891] +- 强制：默认 opt-in 路径用 MindIE-SD TND varlen，host list 传累计 end offsets；只有 env 精确等于 + `ascend_laser_attention` 才改走 prefix K/V slice。resolver 只接受 non-causal、batch=1、四个 packed + key 齐全、Q/K cu shape 相同且至多两 documents、Python-int max length 在 bounds 内，以及 + `[real,pad]` 中 real 不短于 pad；无 padding 的单 document 也接受。Q/K/V 从 BSND squeeze batch + 不复制；Laser 保留 full Q、只裁 K/V，padding-row output 必须由下游忽略。 +- 强制：packed branch 拒绝 metadata 时，如果 caller 因 capability 没构造 mask，backend 必须用合法 + `valid_kv_length` 重建 prefix mask；缺失、非 int 或越界立即报错，不能对 padding 做 unmasked + attention。重建路径仍会扩展 quadratic `full_qk`，所以是 correctness fallback，不是 memory + fast path。Laser 可消费数值型 `input_scale>1`：Q/K/V 同除、softmax scale 乘平方、output 再乘回； + absent/invalid scale 不缩放。模型只应给有硬件数值证据的 exact power-of-two factor。 +- 禁止：把任意 `MINDIE_SD_FA_TYPE` 当作 honored selection；target 对除 exact Laser string 外的合法值、 + typo 和垃圾值都静默走 TND varlen。不得从 capability=true 推断任意 packed layout安全：resolver + 只检查 cu tensor shape、不读其 offset values，真实 producer 必须另证 `[0,used,total]` 内容。 +- 验收：CPU/mock 覆盖 resolver 接受/拒绝矩阵、env dispatch、default caller masked path、fallback + mask/fail-closed、host lists、Laser layout/slice/scaling；真实 NPU 再与 masked FP32/BF16 reference + 对照。target tests 覆盖前者但 mock kernel 不证明 MindIE 数值；NPU SDPA 的同类 `full_qk` churn + 未改。新增 capability 是 classmethod，所有非 `AttentionBackend` duck-typed doubles 也须兼容；target + TeaCache `FakeBackend` 缺该方法并在 cache extractor tests 触发 `AttributeError`，说明 consumer + integration 未闭环。^[PR #5891] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 31083786f..b72d131d8 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5e4b9f8c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 740f45ab` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 08c8549cf..5b0e7c257 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5e4b9f8c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 740f45ab` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index be8d56edb..43e7e2eb7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5e4b9f8c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 740f45ab` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6b7898b5e..3be2d7966 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5e4b9f8c` 验证存在 +- 源码校验：以上路径均已在 `main @ 740f45ab` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 608552335..4f16ec45e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5e4b9f8c`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 740f45ab`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from ae811043 \ - --to 5e4b9f8c \ + --from 5e4b9f8c \ + --to 740f45ab \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 5f2f65020..8c5020e08 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5881", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -63,6 +63,9 @@ confidence: high - Ascend NPU 可选择 RainFusion 稀疏 video tail，并从 BF16 checkpoint 做 online INT8；两者只在 exact T2VA/Ulysses/no-AllGather DLO 配置有完成证据，几何 fallback、TP width 与组合边界见 MMH3-1a/1d。 +- Ascend NPU `FLASH_ATTN` 的 H3 non-Ring packed path 可 opt in mask-free varlen，Laser env 改走 + prefix K/V slice + 256 input scale；shared fallback/metadata 与 target TeaCache regression 见 + MMH3-1g/DIFF-1g，有界 kernel/E2E/HBM 证据见 MMH3-3g。 - ROCm BF16 的 gfx942/MI300X 四卡与 gfx950/MI350 单卡 functional evidence、AITER backend gate、 mutable image 和 support-table 不一致边界见 MMH3-3d；不得从 gfx architecture 名直接扩展 SKU。 - TeaCache 只 hook FL2VA DiT，Ref2VA-only 拒绝；model default、public 入口覆盖、request `quality` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 968a76670..24f77511b 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5863", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/rocm/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5863", "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -152,3 +152,25 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule output/quality；每个 topology 当前仅两次 warmup 后测一个请求，无 repeats/variance。recipe-only diff 无自动回归，4/8 GPU scaling 和首次请求 19% 慢均为单机观察。SM120 显式 CUDNN 合理，因为 target TRTLLM auto-default 只覆盖 compute major 10；这不构成其他 SM120 backend 的比较证据。 + +## MMH3-3g — Ascend mask-free 数字只绑定报告的 H3 packed workload + +- 触发：引用 NPU quadratic mask churn、packed varlen/Laser latency、A3 HBM 或 H3 mask-free E2E。 +- 强制：memory snapshot 只绑定报告的 T2VA packed S=63232、约 100 attention/step×50 steps：旧 + `full_qk` 行产生 3.72/14.90 GiB transient mask、累计 45.6 TiB allocation churn，并使 57 GiB + reserved 中约 33.7 GiB 成为 idle segments；这是 profiler observation，不是所有 shape 的 allocator + 定律。single-kernel 20-iteration average 只覆盖 USP8 per-rank 的四个 T/N/D shape，最大一行 + 63232/7/128 为 170.8→90.6 ms；per-call peak 18.86→0.44 GiB。^[PR #5891] +- 强制：E2E 数字只绑定 8×Ascend 910、batch1/BF16/50 steps/flow shift12/24 FPS、USP8/Ring1/ + HSDP8/TE-TP8/VAE tile-patch8、768p、每格两次均值。varlen→Laser 分别是 T2VA 10 s + 450→337 s、FL2VA 8 s keyframe 316→244 s、Ref2VA 5 s video+audio 642→473 s；Laser+Cache-DiT + 的 226/143/249 s 是叠加另一机制，不归因本 PR。recipe 另报 DLO 下 Ref2VA 13.88 s input→15 s + 1344×768 output 约 45 GB/device；不能与 HSDP E2E topology 合并成同一 capacity/perf case。 +- 禁止：把这些值写成 Ascend/NPU 通用收益，或声称 PR 已证实整请求从旧 mask path 的 speedup；E2E + 表比较的是新 varlen、Laser 和可选 Cache-DiT，没有 old-mask E2E control。rank0 reserved + 40–44 GB 与 recipe DLO 45 GB 也不是所有 rank/任务峰值上界。 +- 验收：复现必须补 exact source commit、immutable environment/image、Atlas/Ascend SKU、输入 artifact/ + prompt/seed、warmup、逐 run raw headers/logs、all-rank memory 与 same-seed output metric。PR body 引用的 + `server_test/{fl2va,ref2va}` scripts 不在 commit，raw profiler/benchmark artifact 也未提交；其 output + video 只覆盖 Ref2VA Laser case，不是数值 quality gate。后续仍需 same-seed E2E comparison 和重抓 + snapshot 证明 churn/reserved 实际下降。^[PR #5891] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index d0a063b2a..4220a1281 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5896", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] confidence: high --- @@ -23,6 +23,7 @@ confidence: high | RainFusion、block sparse、video layout、NPU INT8 | `MMH3-1d` | packed sequence → attention metadata/backend plan；quant config → prefixed linear → loader/post-load | | TRTLLM、ragged packed metadata、SAGE、Skip-Softmax、Blackwell default | `MMH3-1e` | H3 metadata/roles → TRTLLM packed trim/quant gate → platform default | | text encoder、missing q/k/v 或 gate/up、eager load bookkeeping | [MMH3-1f](rules-loading.md#mmh3-1f-text-encoder-eager-load-必须证明每个-source-shard-完整) | `encoder.py::_load_weights` → `pipeline_minimax_h3.py::load_weights` strict report | +| NPU packed varlen、quadratic mask、LaserAttention、prefix K/V | `MMH3-1g` | H3 packed producer → backend capability/metadata → NPU FlashAttention fallback | | FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | | media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | @@ -36,6 +37,7 @@ confidence: high | ROCm、gfx942/gfx950、AITER、BF16、support matrix | [MMH3-3d](rules-deployment.md#mmh3-3d-rocm-support-必须按-sku镜像拓扑和测量协议限界) | support table/footnote → recipe protocol → ROCm backend gate → exact hardware evidence | | DGX Spark、GB10、unified memory、FP8、offload/OOM | [MMH3-3e](rules-deployment.md#mmh3-3e-gb10-unified-memory-容量证据不等于离散-gpu-offload-合同) | single-partition recipe → allocator/header evidence → output probe | | RTX PRO 6000、TP2、Ulysses、2/4/8 GPU、PCIe | [MMH3-3f](rules-deployment.md#mmh3-3f-rtx-pro-6000-scaling-只绑定单机-t2va-协议) | topology → exact warmed T2VA measurements → memory method | +| Ascend packed varlen、LaserAttention、mask churn、E2E/HBM | [MMH3-3g](rules-deployment.md#mmh3-3g-ascend-mask-free-数字只绑定报告的-h3-packed-workload) | H3 opt-in → exact NPU topology/workload → kernel/E2E/memory evidence | ## MMH3-1b — joint video/audio quality 与 offload 兼容性必须一起验收 @@ -138,6 +140,25 @@ confidence: high 来自 review follow-up 外部分支 commit `20cc23ae`，其 artifacts 明确是 combined packed-H3 optimization evidence，并非目标 commit 的隔离实验，不能泛化为稳定 14% 保证。^[PR #5779] +## MMH3-1g — H3 NPU mask-free opt-in 必须保留 Ring 与 malformed-metadata fallback + +- 触发：修改 H3 `_run_packed_attention`、NPU Flash backend capability、packed padding 或 + LaserAttention scale。 +- 强制：H3 在非 Ring 时可因 backend 的 prefix slicing 或 packed-mask-free capability 不构造 mask， + 并发布四个 cu/max key、`valid_kv_length`、`npu_attn_varlen=True` 和 + `laser_input_scale=256.0`。Ring 必须令 opt-in false 并保留 padding mask，因为 aligned rows 是固定 + P2P buffer 合同；CUDA packed varlen/cuDNN prefix slicing 继续走各自 capability，H3 caller 不按 + platform 分支。shared NPU resolver/fallback 细节见 DIFF-1g。 +- 禁止：让 capability method 变成 duck-typed backend 的硬依赖却不更新 doubles/plugins；不能把 + H3 的 `[real,pad]` contract 泛化为 ragged multi-document。Laser 的 256 缩放只针对报告的 fp16 + workspace overflow，其他 backend 忽略；online KV quant path不经过本优化。target TeaCache + `FakeBackend` 缺 method 并在 cache extractor tests 触发 `AttributeError`；必须补 default-safe + capability access 或更新所有实现/doubles 后才算集成闭环。 +- 验收：model contract tests 分别覆盖 prefix-slice、packed-mask-free、普通 backend 与 Ring；backend + tests 再覆盖 malformed fallback 和 Laser math。真实 Ascend parity/内存/延迟证据不能由 CPU mock + 替代；同 seed E2E output comparison 与重新抓 memory snapshot 在 PR 中仍列为 follow-up。 + ^[PR #5891] + ## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 - 触发：H3 `t2va`/`fl2va`/`ref2va`、reference 计数、frame index、canvas、duration 或 From 6c4dae1d0cdff2194af4e19aca27bccf6a3e8900 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 22:39:06 +0800 Subject: [PATCH 082/465] knowledge(vllm-omni): sync 62e239bf (PR #5998) Upstream-Commit: 62e239bf1e7e53240873ada6c18a4ac11ada8828 Upstream-PR: #5998 Knowledge-Delta: rendered example link-integrity boundary --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/docs/generation-and-support-rules.md | 12 +++++++++++- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 23 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6dd0dd178..8f02a4ff2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5e4b9f8c6e503080f16595c578139d37759de1bb + previous_audited_sha: 740f45abcd2e6754d1cdded01ee9901d595ccac5 audited_ref: main - audited_sha: 740f45abcd2e6754d1cdded01ee9901d595ccac5 + audited_sha: 62e239bf1e7e53240873ada6c18a4ac11ada8828 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d004c3710..41ac1f972 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -740f45abcd2e6754d1cdded01ee9901d595ccac5`. The canonical machine baseline +62e239bf1e7e53240873ada6c18a4ac11ada8828`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index de88f96b6..a0461d5ec 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 740f45ab` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 62e239bf` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 439ee26a6..b3c800586 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 740f45ab` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 62e239bf` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b72d131d8..46fec5e8b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 740f45ab` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 62e239bf` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5b0e7c257..c41814292 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 740f45ab` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 62e239bf` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 43e7e2eb7..b262ef56f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 740f45ab` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 62e239bf` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3be2d7966..eea8487c4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 740f45ab` 验证存在 +- 源码校验：以上路径均已在 `main @ 62e239bf` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md index 9fa9ae856..2c9d50dbf 100644 --- a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -4,7 +4,7 @@ created: 2026-08-10 updated: 2026-08-10 type: rule tags: [vllm-omni, docs] -sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987"] +sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998"] --- # Generated docs and supported-model evidence rules @@ -47,6 +47,16 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, `docs/user_guide/examples/`，hook 既不刷新也不删除；quickstart、design 与 feature 文档仍有直接 链接。因此当前链接可达不等于内容会随 source 更新。需要明确迁移链接到 shared task page，或给 stale generated pages 定义删除/刷新策略及 link-integrity test。 +- `diffusers_pipeline_adapter.md` 直接证明该边界：它已被 serving whitelist 排除却仍可访问并需手工 + 修补。source README 位于 `examples/online_serving/diffusers_pipeline_adapter/`，其中 + `../../../docs/user_guide/diffusion/attention_backends.md` 按 source base 正确；相同字面路径若复制到 + `docs/user_guide/examples/online_serving/` 会解析成不存在的 `docs/docs/...`，rendered page 必须用 + `../../diffusion/attention_backends.md` 或经过显式 absolute rewrite。不得假设 source-relative link + 可原样搬到另一个输出目录。^[PR #5998] +- 验收 tracked stale/generated page 时，link checker 必须分别以 source 与 rendered file 的目录为 + base，并跑 clean strict MkDocs 覆盖不在 nav 的可达页面；只测试 generator predicate 或只检查 source + README 不足。该修复报告 strict build 通过但未增加自动 link regression，因此后续手工同步仍可能 + 再漂移。attention backend 段落未改，不能把 link-only diff 当 Diffusers adapter runtime 新证据。 ## DOCGEN-1b — Recipe link 与硬件 checkmark 必须共享可审计证据 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4f16ec45e..a6dcaf6d1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 740f45ab`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 62e239bf`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 5e4b9f8c \ - --to 740f45ab \ + --from 740f45ab \ + --to 62e239bf \ --repo \ --mode report-only ``` From 713fcbdade3ead0c1446bca00794d023d24e35a1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 22:47:03 +0800 Subject: [PATCH 083/465] knowledge(vllm-omni): sync 422d8fd8 (PR #5997) Upstream-Commit: 422d8fd8f9313176802b792e41a30fcd286d6408 Upstream-PR: #5997 Knowledge-Delta: TeaCache attention capability test contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules-attention.md | 10 ++++++---- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 11 ++++++----- 11 files changed, 24 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8f02a4ff2..c79318a4c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 740f45abcd2e6754d1cdded01ee9901d595ccac5 + previous_audited_sha: 62e239bf1e7e53240873ada6c18a4ac11ada8828 audited_ref: main - audited_sha: 62e239bf1e7e53240873ada6c18a4ac11ada8828 + audited_sha: 422d8fd8f9313176802b792e41a30fcd286d6408 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 41ac1f972..cff3fac6a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -62e239bf1e7e53240873ada6c18a4ac11ada8828`. The canonical machine baseline +422d8fd8f9313176802b792e41a30fcd286d6408`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a0461d5ec..af6e8e327 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 62e239bf` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 422d8fd8` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b3c800586..3cf7263ee 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 62e239bf` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 422d8fd8` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index ed2864639..eeb974551 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5891", "PR #5897", vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py] +sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/cache/test_teacache_extractors.py] confidence: high --- @@ -50,6 +50,8 @@ confidence: high - 验收：CPU/mock 覆盖 resolver 接受/拒绝矩阵、env dispatch、default caller masked path、fallback mask/fail-closed、host lists、Laser layout/slice/scaling；真实 NPU 再与 masked FP32/BF16 reference 对照。target tests 覆盖前者但 mock kernel 不证明 MindIE 数值；NPU SDPA 的同类 `full_qk` churn - 未改。新增 capability 是 classmethod，所有非 `AttentionBackend` duck-typed doubles 也须兼容；target - TeaCache `FakeBackend` 缺该方法并在 cache extractor tests 触发 `AttributeError`，说明 consumer - integration 未闭环。^[PR #5891] + 未改。capability 是 classmethod，所有非 `AttentionBackend` duck-typed doubles/plugins 也须兼容； + repo 内 TeaCache `FakeBackend` 已镜像 abstract default，显式返回 false 并恢复 masked-path + extractor coverage。该修复只改 test double，没有给 production consumer 加 default-safe access，也 + 没有证明 third-party registry override 或未继承 base class 的 backend 完成了 capability + census；这些实现仍须继承合同或显式实现 method。^[PR #5891] ^[PR #5997] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 46fec5e8b..44905ac5b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 62e239bf` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 422d8fd8` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c41814292..500bcc377 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 62e239bf` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 422d8fd8` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b262ef56f..c779d3fa0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 62e239bf` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 422d8fd8` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index eea8487c4..076bf3767 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 62e239bf` 验证存在 +- 源码校验：以上路径均已在 `main @ 422d8fd8` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a6dcaf6d1..35383d4d2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 62e239bf`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 422d8fd8`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 740f45ab \ - --to 62e239bf \ + --from 62e239bf \ + --to 422d8fd8 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 4220a1281..e5fd4e622 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5997", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] confidence: high --- @@ -151,13 +151,14 @@ confidence: high platform 分支。shared NPU resolver/fallback 细节见 DIFF-1g。 - 禁止：让 capability method 变成 duck-typed backend 的硬依赖却不更新 doubles/plugins；不能把 H3 的 `[real,pad]` contract 泛化为 ragged multi-document。Laser 的 256 缩放只针对报告的 fp16 - workspace overflow，其他 backend 忽略；online KV quant path不经过本优化。target TeaCache - `FakeBackend` 缺 method 并在 cache extractor tests 触发 `AttributeError`；必须补 default-safe - capability access 或更新所有实现/doubles 后才算集成闭环。 + workspace overflow，其他 backend 忽略；online KV quant path不经过本优化。repo 内 TeaCache + `FakeBackend` 现已实现 capability 并返回 false，所以 extractor tests 保持 abstract-default 的 + masked path；这不等于 production consumer 已有 default-safe access。第三方 duck-typed backend + 仍必须继承 base 或实现该 classmethod，并对 registry override 做 capability census。 - 验收：model contract tests 分别覆盖 prefix-slice、packed-mask-free、普通 backend 与 Ring；backend tests 再覆盖 malformed fallback 和 Laser math。真实 Ascend parity/内存/延迟证据不能由 CPU mock 替代；同 seed E2E output comparison 与重新抓 memory snapshot 在 PR 中仍列为 follow-up。 - ^[PR #5891] + ^[PR #5891] ^[PR #5997] ## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 From 8d5b87cf36e670169362dd2c45367acc4f6f03df Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 22:56:16 +0800 Subject: [PATCH 084/465] knowledge(vllm-omni): sync f8fa86fb (PR #6008) Upstream-Commit: f8fa86fb1fbd6a09e79b59404af9fc568d08bf08 Upstream-PR: #6008 Knowledge-Delta: TTS ratchet decrease semantics --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 12 +++++++++++- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/components/serving/rules.md | 11 +++++++---- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 30 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c79318a4c..bbab51d1d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 62e239bf1e7e53240873ada6c18a4ac11ada8828 + previous_audited_sha: 422d8fd8f9313176802b792e41a30fcd286d6408 audited_ref: main - audited_sha: 422d8fd8f9313176802b792e41a30fcd286d6408 + audited_sha: f8fa86fb1fbd6a09e79b59404af9fc568d08bf08 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index cff3fac6a..41df6497a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -422d8fd8f9313176802b792e41a30fcd286d6408`. The canonical machine baseline +f8fa86fb1fbd6a09e79b59404af9fc568d08bf08`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 249cc98c7..7b632e6ba 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343"] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .pre-commit-config.yaml, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -65,6 +65,16 @@ confidence: high - 验收：当前 main schema/pre-commit、DCO 和 marker collection 通过；篡改摘要、Windows 路径、 shell header 或非法 stability 参数分别被对应 gate 拒绝。 ^[PR #3422] ^[PR #6273] ^[PR #6293] ^[PR #6343] +- pre-commit ratchet 不能把债务下降当成 hook 失败：`actual > budget` 才返回 1， + 相等静默通过， + `actual < budget` 必须返回 0 并打印收紧上限的 reminder。passing hook 需 `verbose: true` + 才能让提示可见；hook 的 `files` 还必须包含 checker 自身，使 budget 修改也会 + 触发审计。下降后未及时收紧会留出 slack，被检测分支只要不超旧上限就可 + 回增；直接提高 budget 也只会通过并打印 slack notice，所以“上限只能下降”仍由 + reviewer 审核。手写 repo-wide constant 还使 verdict 受 merge order 影响，不是 + merge-base-relative 合同。 + 现有 unit test 另外直接断言 count 等于 budget，所以下降但未改 constant 时 hook 会通过， + 而该 test 若被收集仍会失败；两者应统一为同一非递增语义。^[PR #6008] ## OMNI-CI-2b — 并行测试基础设施隔离任务失败和共享状态 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index af6e8e327..be78498a2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 422d8fd8` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f8fa86fb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3cf7263ee..67af6afa6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 422d8fd8` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f8fa86fb` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 44905ac5b..73e6c7788 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 422d8fd8` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f8fa86fb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 500bcc377..8a315e39a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 422d8fd8` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f8fa86fb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c779d3fa0..f48b60ec7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 422d8fd8` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f8fa86fb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 076bf3767..d578d0f7f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 422d8fd8` 验证存在 +- 源码校验：以上路径均已在 `main @ f8fa86fb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index c9db3eb24..f485e76cd 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] +sources: ["Issue #5369", "PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] confidence: high --- @@ -313,8 +313,11 @@ confidence: high `for_diffusion()` 直接绕过 adapter detection，当前 `DiffusionTTSAdapter` 没有生产 subclass。 - 验收：冻结旧 ladder 作为 oracle，覆盖 in-tree pipeline stage、adapter 声明 architecture、 stage 集合等价、entry-arch 范围、无歧义/稳定顺序和 topology guard。`LegacyDetector` 只能缩减； - AST ratchet 的 branch/legacy budget 也只能下降，但 table dispatch 是已固定的漏检形态，不能把 - pre-commit 通过解释为不可绕过。`omnivoice_generator` 未被 pipeline 发出，是保留兼容事实而非 + AST ratchet 中超过 branch/legacy budget 失败，等于上限静默通过，少于上限则通过 + 并显示收紧 reminder；下降本身不能让该 pre-commit hook 红灯，但未同步更新的 exact-equality + pytest 仍会失败。在人工调低 constant 前存在可回增 slack，table dispatch 也是已固定的漏检形态， + 不能把 pre-commit 通过解释为不可绕过。 + `omnivoice_generator` 未被 pipeline 发出，是保留兼容事实而非 可达性证据。^[PR #5682] - adapter 提取保持共享 dispatcher 单一：Ming Flash 的 `validate()` 原样拒绝空 input、过长/非 字符串 instructions、`task_type`/`language`/`x_vector_only_mode`/ @@ -324,7 +327,7 @@ confidence: high `serving_speech.py`。`LEGACY_TTS_DETECTORS` 当前为空，branch/legacy AST ratchet 是 27/0； Ming Flash 只由 adapter 的 `ming_tts` stage key 检测，且 legacy 名不得同时注册 adapter。 验收仍须证明 `ming_tts` stage key 与 Ming dense architecture fallback 的消歧不变。 - ^[PR #5746] ^[PR #5843] + ^[PR #5746] ^[PR #5843] ^[PR #6008] - 证据：CPU oracle/ratchet suite 支撑检测等价；L20X VoxCPM2 只证明 architecture path 到达 warmup，随后因基线同样复现的 vLLM skew 退出，未证明成功 speech endpoint/audio E2E。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 35383d4d2..1bc05cc46 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 422d8fd8`（2026-08-10）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f8fa86fb`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 62e239bf \ - --to 422d8fd8 \ + --from 422d8fd8 \ + --to f8fa86fb \ --repo \ --mode report-only ``` From 8fec55ab5ad79944d63f195fa9ae14f382024c2d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 23:05:50 +0800 Subject: [PATCH 085/465] knowledge(vllm-omni): sync c27623c2 (PR #6000) Upstream-Commit: c27623c29c62365424d4633e2d45f01d9eebf039 Upstream-PR: #6000 Knowledge-Delta: RainFusion irregular-grid protected-tail contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-attention.md | 23 ++++++++++++++++++- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minimax-h3/rules.md | 21 ++++++++++------- 11 files changed, 47 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bbab51d1d..9d47ecca3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 422d8fd8f9313176802b792e41a30fcd286d6408 + previous_audited_sha: f8fa86fb1fbd6a09e79b59404af9fc568d08bf08 audited_ref: main - audited_sha: f8fa86fb1fbd6a09e79b59404af9fc568d08bf08 + audited_sha: c27623c29c62365424d4633e2d45f01d9eebf039 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 41df6497a..c5d2bc62b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f8fa86fb1fbd6a09e79b59404af9fc568d08bf08`. The canonical machine baseline +c27623c29c62365424d4633e2d45f01d9eebf039`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index be78498a2..ca72d9a7f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f8fa86fb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c27623c2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 67af6afa6..a57597951 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f8fa86fb` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c27623c2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index eeb974551..ff675b35f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/cache/test_teacache_extractors.py] +sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py] confidence: high --- @@ -55,3 +55,24 @@ confidence: high extractor coverage。该修复只改 test double，没有给 production consumer 加 default-safe access，也 没有证明 third-party registry override 或未继承 base class 的 backend 完成了 capability census；这些实现仍须继承合同或显式实现 method。^[PR #5891] ^[PR #5997] + +## DIFF-1h — RainFusion irregular tail 由 MindIE-SD 保护，不能 padding 伪对齐 + +- 触发：修改 `RAINFUSION_ATTN`、`rf_v2`、`VideoTokenLayout`、block size、MindIE-SD + 依赖或 packed video shape。 +- 强制：resolver 传入真实 `prefix_len`、`latent_shape=[t,h,w]` 和 + `used_len=prefix_len+t*h*w`；只裁掉 document-0 后的物理 padding，kernel 输出再补零回 + query shape。video rows 不再要求 128 整除；新 MindIE-SD 在空间重排后把不规则的 + 真实 video suffix 提升到 always-kept segment，使余下 video 按 128-row block 稀疏。 + 不能用人工 padding 替代，因为 `rf_v2` 不消费 padding mask，pad key 会污染 softmax。 +- 边界：`sparsity<=0`、未到 `start_step`、skip layer、无/未声明 BSND layout、无 + video layout/无 `max_seqlen_q`、`prefix+t*h*w` 不闭合 document 0，或 video 少于 + 32×128 rows 均保持 FlashAttention dense fallback；显式错 layout、causal 或 Ring>1 + 仍 fail closed。非 8×8 空间网格的 residual 被 always-kept，因而 realized sparsity 低于 + nominal；潜空间 h/w 为 8 的倍数（输入高宽为 256 的倍数）仍是性能首选。 +- 禁止：只因 `mindiesd` 可 import 就断言 irregular-tail 合同可用；当前 availability + 只检查 module 存在，没有 version/feature gate，旧 `rf_v2` 与新 planner 组合仍有风险。 +- 验收：CPU plan 同时接受 128-aligned 与 irregular grid，并保持所有其他 fallback/ + rejection；真实 Ascend + updated MindIE-SD 用 irregular grid 对 dense reference。目标 NPU test + 仅以 `sparsity=0` 证明 protected-tail 几何下全 block 等价，不证明稀疏选择的质量、 + realized sparsity 或加速。^[PR #6000] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 73e6c7788..7f010840d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f8fa86fb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c27623c2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8a315e39a..158cee11d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f8fa86fb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c27623c2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f48b60ec7..f7b67c6c6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f8fa86fb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c27623c2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d578d0f7f..3e0e2264f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f8fa86fb` 验证存在 +- 源码校验：以上路径均已在 `main @ c27623c2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1bc05cc46..09673c2ab 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f8fa86fb`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c27623c2`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 422d8fd8 \ - --to f8fa86fb \ + --from f8fa86fb \ + --to c27623c2 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index e5fd4e622..bbbcacd8f 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5997", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] confidence: high --- @@ -92,25 +92,30 @@ confidence: high 当前自动测试没有 MUSA dispatch/parity，CPU reference 只验证 96 维旋转和 32 维 passthrough。^[PR #5881] -## MMH3-1d — RainFusion 只在完整、对齐的 H3 video tail 上稀疏 +## MMH3-1d — RainFusion 只在完整 H3 video tail 上稀疏 - 触发：修改 `RAINFUSION_ATTN`、`block_sparse`、packed video geometry、attention layout、 denoise step/layer skip 或 NPU backend resolution。 - 强制：显式选择时所有平台在构造模型前检查 backend platform/dependency；RainFusion 仅支持 Ascend NPU、要求 MindIE-SD，且不兼容 Ring（用 Ulysses）。H3 attention 显式声明 BSND， `VideoTokenLayout(prefix_len, latent_grid)` 必须证明 video 是 packed document 0 的 tail。 -- 强制：只有 sparsity>0、step>=start_step、非 skip layer、video>=32×128 rows 且 video rows - 为 128 的倍数时调用 `rf_v2`；prefix/text/reference/audio 保持 dense。无 layout、无 - `max_seqlen_q`、长度不闭合、短序列、未声明 layout 或未对齐 geometry 一律走 NPU Flash dense - fallback，不能 padding，因为 kernel 忽略 mask，padding key 会改变 softmax denominator。 +- 强制：只有 sparsity>0、step>=start_step、非 skip layer 且 video>=32×128 rows 时调用 + `rf_v2`；prefix/text/reference/audio 保持 dense。无 layout、无 `max_seqlen_q`、长度不闭合、 + 短序列或未声明 layout 仍走 NPU Flash dense fallback。video rows 不被 128 整除时 + 也传给 updated MindIE-SD，由它将 irregular real-video suffix 纳入 always-kept 段； + vLLM 不 padding。shared planner 和 dependency 边界见 DIFF-1h。 - 禁止：把 nominal sparsity 当 realized sparsity或质量保证；未声明 layout 时不能让 sparse path 假设 BSND 而 dense fallback 解释成 BNSD。INT8、RainFusion、no-AllGather DLO 可组合，但 online quantization 不得与 DLO+AllGather 组合；本 PR 没有证明 HSDP 或其他 quantizer 的组合语义。 -- 验收：CPU plan tests 覆盖 aligned/misaligned、tail closure、min length、layout 和 skip/step； +- 验收：CPU plan tests 覆盖 aligned/irregular 均进 sparse plan、tail closure、min length、layout + 和 skip/step； NPU 条件数值测试以 `sparsity=0` 直接调用 kernel 对 dense reference，mean relative error 阈值 为 `2e-3`；它不证明 `sparsity=0.8` 质量 parity。PR 的 Atlas 800I A3 8×NPU、 CANN 9.0.1、T2VA 209-frame 三种生成仅证明这些 exact 配置能完成；视频样例不是质量阈值， - 没有 latency/repeats，不能宣称稳定加速或 FL2VA/Ref2VA 支持。^[PR #5706] + 没有 latency/repeats，不能宣称稳定加速或 FL2VA/Ref2VA 支持。1344×768 + H3 grid `(62,24,42)` 的 62496 video rows 只在 CPU plan test 证明会进 sparse plan； + updated-MindIE NPU 对照用更小的 `(4,16,10)` grid 且 `sparsity=0`，仍不证明 0.8 + 的质量或性能。^[PR #5706] ^[PR #6000] ## MMH3-1e — H3 TRTLLM 必须从 packed 结构裁掉 padding 并隔离短序列 role From 99156ab0d7e2d146901cbfd713dd9f311f76870e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 23:23:24 +0800 Subject: [PATCH 086/465] knowledge(vllm-omni): sync ee118ab4 (PR #4583) Upstream-Commit: ee118ab492c2c6057169c36c2320f7c4f06219fa Upstream-PR: #4583 Knowledge-Delta: per-replica stage fault-isolation contract --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 3 +- .../serving/rules-fault-isolation.md | 56 +++++++++++++++++++ .../vllm-omni/components/serving/rules.md | 5 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +- 11 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-fault-isolation.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9d47ecca3..3e2ce0b42 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f8fa86fb1fbd6a09e79b59404af9fc568d08bf08 + previous_audited_sha: c27623c29c62365424d4633e2d45f01d9eebf039 audited_ref: main - audited_sha: c27623c29c62365424d4633e2d45f01d9eebf039 + audited_sha: ee118ab492c2c6057169c36c2320f7c4f06219fa # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c5d2bc62b..fcf71dd72 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c27623c29c62365424d4633e2d45f01d9eebf039`. The canonical machine baseline +ee118ab492c2c6057169c36c2320f7c4f06219fa`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ca72d9a7f..489b4ae50 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c27623c2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ee118ab4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a57597951..5ab5ccf57 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c27623c2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ee118ab4` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7f010840d..f85b26c32 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c27623c2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ee118ab4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 158cee11d..6b934d2a2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c27623c2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ee118ab4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f7b67c6c6..98d7a1341 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c27623c2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ee118ab4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3e0e2264f..13b8a2136 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c27623c2` 验证存在 +- 源码校验：以上路径均已在 `main @ ee118ab4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -29,4 +29,5 @@ sources: [] |---|---| | 理解入口到 engine 的边界 | [architecture](architecture.md) | | 根据 PR 描述直达 request contract、streaming format、artifact readiness、factory lifecycle 或 metrics 的规则组与第一批源码 | [rules 与代码地图](rules.md) | +| stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-fault-isolation.md b/knowledge/repos/vllm-omni/components/serving/rules-fault-isolation.md new file mode 100644 index 000000000..3cb08da16 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-fault-isolation.md @@ -0,0 +1,56 @@ +--- +title: "Serving replica fault-isolation 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #4583", vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py] +confidence: high +--- + +# Serving replica fault-isolation 规则 + +## SERV-5f — stage death 按 replica 隔离，request fatal 不等于 process fatal + +- 触发：stage poll/submit 抵达 `EngineDeadError`、replica 集合变化、dispatch timeout、 + `/health`、`errored`/`is_running`、request error queue 或 external restart policy。 +- 强制：LLM 和 diffusion poll 均在单 replica 边界捕获 `EngineDeadError`，先 shutdown client + 再将 pool slot 置 `None`，使 live set 不再 poll/dispatch 到它，而 orchestrator loop 继续。 + 若同 stage 仍有非 `None` slot，只失败绑定到死 replica 的 request；若一个也不剩， + 失败 `stage_submit_ts` 显示曾进入该 stage 的所有 active request。该标记/绑定只在整请求 + cleanup 清除，所以 last-replica 边界是 routed-through history，不是已证明精确的 current + in-flight 集合；已完成上游 stage 但仍 active 的请求需单独回归。 +- 强制：所有可达 dispatch 边界（stage-0 add、streaming update、归属 parent 的 CFG + companion、按每个真实 downstream stage id 处理的 async-chunk prewarm、inter-stage forward） + 用 thunk 包起选 replica 和 await，使同步 setup 与异步 submit 都在 guard 内。 + `StageUnavailableError` 只表示空 live set、被驱逐 slot 或 dispatch timeout，转为该 request + 的 5xx；不相关 `RuntimeError` 继续 fail fast。submit 中遇 `EngineDeadError` 须立即驱逐 + 绑定 replica，不能等 poll。 +- 强制：带 `request_id` 的 `ErrorMessage(fatal=True)` 仍是 request-scoped，AsyncOmni 送到 + 对应 queue 后继续 output loop；只有不带 request id 的 fatal error 才是 engine-wide + teardown。失败 request 统一经 `_cleanup_request_ids(..., abort=True)` 释放 running counter、 + PD/CFG state 和全部 pool bindings，并 abort 存活 stage 上的剩余工作；clients traversal 跳过 + `None` hole。 +- 强制：`errored` 和 `is_running` 只反映 orchestrator process/thread。`check_health` 对每个 + 非空 pool 主动调 client health（包括 diffusion `proc.is_alive()`）；至少一个被证明存活时 + stage healthy，全灭时 readiness 503 但 API process 仍可应答。`None`、无 callable health 或 + health 抛 `EngineDeadError` 都计 dead；`len(pool.clients)==0` 的 zero-slot pool 不计 dead。 + poll handler 的 survivor 判定只看非 `None` slot，与 readiness 的主动探测不是原子快照； + 多 replica 近同时死亡会随各 poll 驱逐而逐步收敛。 +- 禁止：把 readiness 503 解释为 process 已死；把同一 endpoint 当 liveness restart probe + 无条件重启；捕获所有 `RuntimeError`；只 pop request state 而遗留计数器、CFG/PD 或 + 其他 pool binding；用 init-time `engine.stage_clients` snapshot 代替动态 live pool。 +- 验收：CPU/mock 覆盖 LLM/diffusion poll death、两 replica 中一个驱逐后另一请求保留、 + last-replica、每个 dispatch 竞态/归属、统一 cleanup 和不相关异常不被吞；再增加 + “已完成上游 stage 后该 stage 最后 replica 死亡”回归。硬件需真正启动多 replica， + 杀一个后证明 health 200 且 survivor 可服务，再杀最后一个证明 503 但端口可达。 + target unit 的多 replica/stage 是 fake clients，且未直接用一死一活组合调 + `check_health`；review 另报告单一 H20 两 replica stage-1 SIGKILL 后 health 200 与 + voice-clone request 成功。提交内新增的 OOM survival assertion 只证明请求得到 5xx，且 + `/health` 在 15 秒内持续可达并返回 200 或 503；last-replica 后精确 503 来自 PR + 报告的硬件运行，不是该断言的强制条件。^[PR #4583] +- 边界：驱逐不 respawn，暂态压力消失后不恢复 capacity，recovery test 仍 skip； + 由外部编排决定 restart。collective RPC 的目标迭代已读 live pools，但 sleep/wake 的 ACK + 计数、comprehension/diffusion metadata 查找和 PD bootstrap metadata 仍读 init-time + `stage_clients`，动态成员语义需另行收敛。本 pin 没有新增与 readiness 分开的 + liveness endpoint 或部署文档。^[PR #4583] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index f485e76cd..2bec726e1 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] confidence: high --- @@ -24,6 +24,7 @@ confidence: high | `chat_template_kwargs`、raw HTTP/SDK `extra_body`、text/audio modalities、choices、空音频 | `chat-multimodal-contract`：`SERV-4c` + 命中模型规则 | upstream `ChatCompletionRequest` → `serving_chat.py::{OmniOpenAIServingChat._preprocess_chat,OmniOpenAIServingChat.chat_completion_full_generator,OmniOpenAIServingChat._create_text_choice,OmniOpenAIServingChat._create_audio_choice}` | | endpoint restriction、route/app-state guard、capability、公开 400 | `endpoint-capability`：`SERV-4c`, `SERV-4d`, `SERV-5d` | endpoint policy → `api_server.py::build_app` assembled app → public handler | | sleep/wake、partial stage/tag、idempotency、ACK、generation admission | `engine-lifecycle`：`SERV-5a`, `SERV-5b` | `entrypoints/async_omni.py::{AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni.generate}` → `worker/base.py::{handle_sleep_task,handle_wake_task}` / `diffusion/worker/diffusion_worker.py` | +| stage/replica death、`EngineDeadError`、`StageUnavailableError`、health/readiness、eviction | `engine-lifecycle`：[SERV-5f](rules-fault-isolation.md#serv-5f-stage-death-按-replica-隔离request-fatal-不等于-process-fatal) | `engine/stage_pool.py` live set → `engine/orchestrator.py` poll/dispatch/cleanup → `entrypoints/{omni_base,async_omni}.py` error transport | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | | request-level LoRA、AR-only/multistage、streaming input update、stage cardinality | `request-contract`：`SERV-4l` | `serving_chat.py` LoRA resolve → `AsyncOmni.generate` → `AsyncOmniEngine._build_add_request_message` → stage-0 input processor | @@ -41,7 +42,7 @@ confidence: high | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c` + 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d`, `SERV-5d` | -| `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection | `SERV-5a`, `SERV-5b`, `SERV-5c`, `SERV-5e` | +| `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection、replica fault isolation | `SERV-5a`, `SERV-5b`, `SERV-5c`, `SERV-5e`, `SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6c` | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`, `SERV-4b`, `SERV-4c`, `SERV-4d`, `SERV-4e`, `SERV-4f`, `SERV-4g`, `SERV-4h`, `SERV-4l` | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`, `SERV-4j`, `SERV-4k` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 09673c2ab..b3b812883 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c27623c2`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ee118ab4`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from f8fa86fb \ - --to c27623c2 \ + --from c27623c2 \ + --to ee118ab4 \ --repo \ --mode report-only ``` From 726ecb7a8501ab95df22538122acfe11f4cbffd9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 23:34:20 +0800 Subject: [PATCH 087/465] knowledge(vllm-omni): sync 53afbf3a (PR #5175) Upstream-Commit: 53afbf3a0c9b104838fa8fc9940d38fa7935acec Upstream-PR: #5175 Knowledge-Delta: Voxtral TTS graph and host-sync hot-path contract --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +- .../vllm-omni/models/voxtral-tts/_index.md | 7 +- .../vllm-omni/models/voxtral-tts/rules.md | 68 +++++++++++++++++++ 11 files changed, 84 insertions(+), 15 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/voxtral-tts/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3e2ce0b42..8bc7e4fb6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c27623c29c62365424d4633e2d45f01d9eebf039 + previous_audited_sha: ee118ab492c2c6057169c36c2320f7c4f06219fa audited_ref: main - audited_sha: ee118ab492c2c6057169c36c2320f7c4f06219fa + audited_sha: 53afbf3a0c9b104838fa8fc9940d38fa7935acec # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fcf71dd72..bac52e7df 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ee118ab492c2c6057169c36c2320f7c4f06219fa`. The canonical machine baseline +53afbf3a0c9b104838fa8fc9940d38fa7935acec`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 489b4ae50..87cd40133 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ee118ab4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 53afbf3a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5ab5ccf57..49dd554fb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ee118ab4` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 53afbf3a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f85b26c32..15fb7b126 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ee118ab4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 53afbf3a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6b934d2a2..81b050b0c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ee118ab4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 53afbf3a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 98d7a1341..86c910727 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ee118ab4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 53afbf3a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 13b8a2136..7bbde4c17 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ee118ab4` 验证存在 +- 源码校验：以上路径均已在 `main @ 53afbf3a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b3b812883..8a0ca260b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ee118ab4`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 53afbf3a`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from c27623c2 \ - --to ee118ab4 \ + --from ee118ab4 \ + --to 53afbf3a \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/voxtral-tts/_index.md b/knowledge/repos/vllm-omni/models/voxtral-tts/_index.md index 987113b98..b4e60e4e5 100644 --- a/knowledge/repos/vllm-omni/models/voxtral-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/voxtral-tts/_index.md @@ -1,15 +1,15 @@ --- title: "Voxtral TTS（Mistral 两 stage 流式 TTS）" created: 2026-07-21 -updated: 2026-07-31 +updated: 2026-09-02 type: index tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/voxtral_tts/, vllm_omni/deploy/voxtral_tts.yaml] +sources: ["PR #5175", vllm_omni/model_executor/models/voxtral_tts/, vllm_omni/deploy/voxtral_tts.yaml] --- # Voxtral TTS -以下事实在 `main @ 807db6ef` 复核（源码派生页,尚无本模型的运行经验沉淀）。 +以下事实在 `main @ 53afbf3a` 复核。 ## 名称与范围 @@ -40,4 +40,5 @@ sources: [vllm_omni/model_executor/models/voxtral_tts/, vllm_omni/deploy/voxtral - 审查 voxtral_tts 的 mistral 加载三件套一致性、acoustic CUDA-graph 或 async-chunk 改动；XPU 路径回归检查同一 deploy YAML 的 platform override。 +- hot-path graph/sync/cache 边界见 [Voxtral-TTS rules](rules.md)。 - 语义验收见 [model-validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/voxtral-tts/rules.md b/knowledge/repos/vllm-omni/models/voxtral-tts/rules.md new file mode 100644 index 000000000..27e8c21ac --- /dev/null +++ b/knowledge/repos/vllm-omni/models/voxtral-tts/rules.md @@ -0,0 +1,68 @@ +--- +title: "Voxtral-TTS hot-path 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models] +sources: ["PR #5175", vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py, vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py, vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py, vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_tokenizer.py, tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py] +confidence: high +--- + +# Voxtral-TTS hot-path 规则 + +只有 `VOXTTS-数字字母` 是可审计规则 ID。 + +## VOXTTS-1a — variable-output full graph 与 inner acoustic graph 分层 + +- 触发：Voxtral stage-0 FlowMatching、stage-1 tokenizer decode、 + `supports_cudagraph_full`、inner capture/replay、`--enforce-eager` 或 batch bucket。 +- 强制：stage-1 `audio_tokenizer.forward()` 返回 runtime-length per-request list，故 + `supports_cudagraph_full=False`，不能让 FULL wrapper 冻结 capture-time 长度；默认 YAML + 已令 stage-1 eager，此 opt-out 主要约束其他配置/未来 override。stage-0 inner wrapper 只捕获 + 固定 bucket 的 greedy semantic argmax + CFG Euler ODE；enforce-eager、未 warm、超出/缺 bucket， + 或 outer stream 正在 capture 时均走 eager `compute_mm_logits`，禁止 nested graph replay。 +- 强制：replay 前清零/pad static input，逐 request 写 CFG、其余填 1.2，并原位生成 fresh noise； + 输出 clone 后裁到真实 batch。ODE 把 step-invariant `llm_projection` 提出循环；time projection + 与 dt 在 eager path 按 dtype lazy cache，graph path 按 capture device/dtype 预生成。 +- 禁止：把 stage-1 opt-out 说成整族禁用 CUDA graph；把 mutable static buffers 当并发安全； + weights/module device 改变后继续复用旧 schedule。当前 cache 只以 dtype 为 key，不含 device/ + weight generation，也不是 registered buffer；单-device 串行 runner 不证明 module move、 + hot weight update 或同实例并发安全。 +- 验收：固定 weights/input/noise/CFG 对比 eager/graph codes 与 EOS；覆盖 exact/padded/oversize + bucket、fresh noise、per-request CFG、outer-capture fallback、stage-1 动态输出和 cache invalidation。 + target synthetic test只随 pre-projected velocity signature 更新；没有新增上述 opt-out/fallback、 + invalidation 或真实 checkpoint 数值 parity 回归。^[PR #5175] + +## VOXTTS-1b — 合并 host sync 仍须保留 device/empty 语义 + +- 触发：packed header、EOA cutting point、chunk padding、waveform logging、ALiBi/fake-EOS cache。 +- 强制：CUDA `input_ids` 整体 D2H 一次并从 CPU view 读每项 header，token slice 仍取原 tensor； + 这替代逐 request 两次 `.item()`，但也复制 payload，且 `is_cuda` guard 不覆盖 XPU。EOA 在 + device 为每项求 `(any,argmax)` 后一次 stacked D2H；无 EOA 用原长度。chunks 用 + `pad_sequence` 批量 decode 并按原长裁回。 +- 强制：ALiBi 是随 module device 的 non-persistent contiguous FP32 buffer；SDPA 只转 query + dtype。fake-EOS 常量按 device cache。waveform 已无条件 detach/copy CPU 后，WARNING gate + 才决定是否做 CPU min/max；这不消除 waveform D2H，通常 WARNING 开启时仍执行 reductions。 +- 禁止：宣称完全无 host sync，或把 Voxtral-local 优化外推为共享 TTS runtime。EOA 新路径在 + 检查 `has` 前总调用 `argmax`；零长度 codes 会抛错，而旧路径进入显式 empty-result 分支， + 所以不能声称严格等价。 +- 验收：多 request 覆盖 EOA 首/中/末/缺失、零长度、空列表和变长 chunks，对比旧实现的 exact + cutting/output；FlashAttention/SDPA 检查 dtype/device。用 profiler 分离 D2H/H2D 与 kernel， + 不用 wall-clock 单独归因。^[PR #5175] + +## VOXTTS-1c — per-request CFG metadata 与 batch 严格对齐 + +- 触发：`sampling_extra_args`、`cfg_alpha` 或 graph static CFG buffer。 +- 强制：metadata 规范为恰好 B 项；缺 list、缺项、`None` 或缺 key 均逐 request 填 default， + 额外项 fail closed；传入 eager/graph 的 tensor 必须为 `(B,)`。 +- 禁止：短 list 的 `(1,)` 向 B>1 广播、空 list shape mismatch、或对 `None` 直接 `.get`。 +- 验收：覆盖 missing/empty/short/None/missing-key/exact/overlong，并在 B>1 断言独立 CFG。 + target 仍只遍历已提供项；review 指出的错误广播/失败未修复且无回归测试。^[PR #5175] + +## 性能证据边界 + +- 作者 A/B 使用 RTX 6000 Ada、vLLM 0.25、全局 `--enforce-eager` 和可变的 `main`，未给 + checkpoint/prompt/request count/raw artifact；它不验证任何 graph 改动。reviewer 的 exact-base + L20X + vLLM 0.26、默认 YAML、本地 snapshot、每 leg 两个 server/各 13 warm requests，以输出 + bytes 归一后只支持约 3% 的方向性改善。两组都无 profiler、逐优化 ablation 或质量 parity， + 不可泛化吞吐/延迟数字，也不能把 raw latency 用于变长音频比较。^[PR #5175] From 236b667b5ea94c41c98a32b22ca2339205c9eba8 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 23:45:58 +0800 Subject: [PATCH 088/465] knowledge(vllm-omni): sync 12a5f6fb (PR #5067) Upstream-Commit: 12a5f6fbf3ac08ffeec8cb4a2fc05344b8d1e4cf Upstream-PR: #5067 Knowledge-Delta: Step-Audio2 device-resident streaming and sync boundary --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +- .../vllm-omni/models/step-audio2/_index.md | 5 +- .../models/step-audio2/architecture.md | 5 +- .../vllm-omni/models/step-audio2/rules.md | 57 +++++++++++++++++++ 12 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/step-audio2/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8bc7e4fb6..32fa2885d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ee118ab492c2c6057169c36c2320f7c4f06219fa + previous_audited_sha: 53afbf3a0c9b104838fa8fc9940d38fa7935acec audited_ref: main - audited_sha: 53afbf3a0c9b104838fa8fc9940d38fa7935acec + audited_sha: 12a5f6fbf3ac08ffeec8cb4a2fc05344b8d1e4cf # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index bac52e7df..7f28287d6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -53afbf3a0c9b104838fa8fc9940d38fa7935acec`. The canonical machine baseline +12a5f6fbf3ac08ffeec8cb4a2fc05344b8d1e4cf`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 87cd40133..4aaa9d180 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 53afbf3a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 12a5f6fb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 49dd554fb..5796c01f3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 53afbf3a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 12a5f6fb` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 15fb7b126..baf53e016 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 53afbf3a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 12a5f6fb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 81b050b0c..56ef87e72 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 53afbf3a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 12a5f6fb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 86c910727..b5bc51fbf 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 53afbf3a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 12a5f6fb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7bbde4c17..b0165434d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 53afbf3a` 验证存在 +- 源码校验：以上路径均已在 `main @ 12a5f6fb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8a0ca260b..796d27be6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 53afbf3a`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 12a5f6fb`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from ee118ab4 \ - --to 53afbf3a \ + --from 53afbf3a \ + --to 12a5f6fb \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/step-audio2/_index.md b/knowledge/repos/vllm-omni/models/step-audio2/_index.md index d49873fef..a968fd9fa 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/_index.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/_index.md @@ -4,12 +4,12 @@ created: 2026-07-21 updated: 2026-09-02 type: index tags: [vllm-omni, models] -sources: ["PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py] +sources: ["PR #5067", "PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py] --- # Step-Audio2 -以下事实在 `main @ fc8946fc` 复核。 +以下事实在 `main @ 12a5f6fb` 复核。 ## 名称与范围 @@ -50,6 +50,7 @@ sources: ["PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/cod | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | token 过滤桥、25+3 lookahead、流式状态 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | +| 设备常驻 token、ragged 长度同步与性能证据边界 | [rules](rules.md) | Step-Audio2 专属审查规则 | ## 配置与 checkpoint 差异（三变体） diff --git a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md index 3f4aab473..1ef710ff4 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md @@ -4,12 +4,13 @@ created: 2026-07-21 updated: 2026-09-02 type: architecture tags: [vllm-omni, models] -sources: ["PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py] +sources: ["PR #5067", "PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py] --- # Step-Audio2 架构 -事实在 `main @ fc8946fc` 复核;变体/入口速览见 [index](_index.md)。 +事实在 `main @ 12a5f6fb` 复核;变体/入口速览见 [index](_index.md)， +设备边界与同步验收见 [rules](rules.md)。 ## 模型专有部分与共享模块的边界 diff --git a/knowledge/repos/vllm-omni/models/step-audio2/rules.md b/knowledge/repos/vllm-omni/models/step-audio2/rules.md new file mode 100644 index 000000000..320b801d2 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/step-audio2/rules.md @@ -0,0 +1,57 @@ +--- +title: "Step-Audio2 设备边界与流式性能规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models] +sources: ["PR #5067", vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, tests/model_executor/models/step_audio2/test_step_audio2_token2wav_async_chunk.py, tests/platforms/npu/test_step_audio2_token2wav.py] +--- + +# Step-Audio2 设备边界与流式性能规则 + +以下规则在 `main @ 12a5f6fb` 复核；主要约束 Step-Audio2 的 thinker/token2wav +路径。共享 `Token2Wav` core 的 blast radius 在下文显式列出，runtime 的通用规则 +仍归组件 owner。 + +## STEPA2-1a：流式 token 的设备常驻与所有权 + +- 触发：修改 `_forward_async_chunk`、`stream_chunk_for`、设备/dtype 转换或 + `_StreamState` 生命周期。 +- 必须：外层用 `numel()` 判空，非空 tensor 保持为 tensor 传到 + `stream_chunk_for`；内层整形为 `[1, T]`、转到 core device/ + `int32` 后无条件 `clone()`。`clone` 是所有权边界：`.to()` 在已匹配 + device/dtype 时可返回别名，外部 `inference_chunk` 不得污染调用方 + `input_ids`。列表入口仍需兼容 MiniCPM-o 等现有 caller。 +- 不得声称“零拷贝/零传输”：设备已匹配时仍有 clone 分配，设备不同时 + 仍有一次设备传输，最终 waveform 仍通过 `detach().cpu().contiguous()` + 回到 host。该变更只移除 Step-Audio2 token 的 D2H Python list 再 H2D 往返。 +- 验收：添加真实 tensor 分支测试，让 fake `inference_chunk` 就地改写 + 参数并证明 caller tensor 未变；覆盖 device/dtype/shape、list 兼容、 + 空 EOF、last/error 后状态重置。CPU 外的 CUDA/NPU/XPU 结论需各平台 + 证据，不得从通用 `.to()` 推导已验证。 + +## STEPA2-1b：ragged audio-feature 长度只做一次 host 同步 + +- 触发：修改 thinker `_process_audio_input` 中 encoder 输出的 ragged 切片。 +- 必须：encoder 后先将整批 `audio_feature_lens.tolist()` 一次转为 + Python 整数，再按原 batch 顺序切片；长度、数量和空 batch 语义不变。 + 该代码位于 multimodal-encoder 的 eager prefill 路径，不得移入 decode + CUDA graph。 +- 边界：`.tolist()` 仍是一次 host 同步；CUDA/XPU/NPU 上都不应表述为 + “无同步”，CPU 上则没有 accelerator 同步问题。本次无测试改动， + 现有 async-chunk 测试也未证明 tensor 参数、设备常驻或非别名。 + +## 性能证据的可用边界 + +- PR 作者报告 ASR 8 并发 D2H 计数 `17→2`、transcript 字节相同， + 但未给 vLLM/vLLM-Omni SHA、计数脚本、原始产物或硬件；只可用于 + 支持“同步次数方向下降”，不可作通用输出等价或性能基线。 +- reviewer 在 L20X、vLLM 0.26、TP2、默认 async YAML 上做的精确 A/B + 为 3 次/组，中位数 `2.87 s` 与 `2.88 s`、输出同为 `316844 B`； + 它支持该情景下“未见回归、未见可测加速”。TTS 不走 thinker + audio-input 分支，且 `max_num_seqs=1` 无 batch 压力，因此不能验证 + `audio_feature_lens.tolist()` 改动或并发收益。 +- blast radius：thinker 改动仅 Step-Audio2；`stream_chunk_for` core 也被 + MiniCPM-o 4.5/NPU wrapper 使用。MiniCPM 显式传 list，继续走旧分支； + Step-Audio2 NPU wrapper 转发 tensor 并继承新分支。评审共享 core 时 + 不得把两者当作同样的性能受益方。 From a18d4ee5787966c93be6eb624db25558674f9ee0 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 2 Sep 2026 23:57:34 +0800 Subject: [PATCH 089/465] knowledge(vllm-omni): sync 4f9b6507 (PR #5202) Upstream-Commit: 4f9b65074dead2e89f8ace4798a92f2370aec85b Upstream-PR: #5202 Knowledge-Delta: Qwen3-TTS incremental decode and state lifecycle --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +- .../vllm-omni/models/qwen3-tts/_index.md | 9 +- .../repos/vllm-omni/models/qwen3-tts/rules.md | 88 ++++++++++++++++++- 11 files changed, 104 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 32fa2885d..8b89e1861 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 53afbf3a0c9b104838fa8fc9940d38fa7935acec + previous_audited_sha: 12a5f6fbf3ac08ffeec8cb4a2fc05344b8d1e4cf audited_ref: main - audited_sha: 12a5f6fbf3ac08ffeec8cb4a2fc05344b8d1e4cf + audited_sha: 4f9b65074dead2e89f8ace4798a92f2370aec85b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7f28287d6..169bd8c82 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -12a5f6fbf3ac08ffeec8cb4a2fc05344b8d1e4cf`. The canonical machine baseline +4f9b65074dead2e89f8ace4798a92f2370aec85b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4aaa9d180..bf5982fc1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 12a5f6fb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4f9b6507` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5796c01f3..70378f399 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 12a5f6fb` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4f9b6507` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index baf53e016..ba256a55a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 12a5f6fb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4f9b6507` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 56ef87e72..d2dc9368c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 12a5f6fb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4f9b6507` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b5bc51fbf..a18585111 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 12a5f6fb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4f9b6507` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b0165434d..7d77443f5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 12a5f6fb` 验证存在 +- 源码校验：以上路径均已在 `main @ 4f9b6507` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 796d27be6..c7620b50b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 12a5f6fb`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4f9b6507`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 53afbf3a \ - --to 12a5f6fb \ + --from 12a5f6fb \ + --to 4f9b6507 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md index a598632f1..97cf879a4 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5608", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py] +sources: ["PR #5157", "PR #5202", "PR #5608", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py] confidence: high --- @@ -12,7 +12,9 @@ confidence: high ## 名称与范围 -- 正式 owner：Qwen3-TTS serving、ref audio、artifact cache 请求语义与模型专有 NPU patch。 +- 正式 owner：Qwen3-TTS serving、ref audio、artifact cache、async Code2Wav + 增量解码/状态生命周期与模型专有 NPU patch。事实在 `main @ 4f9b6507` + 复核。 - serving 入口：`vllm_omni/entrypoints/openai/serving_speech.py`。 - NPU tokenizer patch：`vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py`；共享 RoPE layout helper 在 `vllm_omni/platforms/npu/layers/rotary_embedding.py`，此 pin 上只 @@ -21,7 +23,8 @@ confidence: high ## 什么时候查这里 -- 审查 `x_vector_only_mode`、ICL、`ref_audio` artifact-only reuse、engine 存活性，或 NPU +- 审查 `x_vector_only_mode`、ICL、`ref_audio` artifact-only reuse、incremental + Code2Wav、segmented CUDA Graph、engine 存活性，或 NPU tokenizer RoPE 的 BNSD/BSND shape fallback。 - 描述直达源码与具体不变量见 [Qwen3-TTS rules](rules.md#direct-代码快速入口)；模型家族结构见 [Qwen-Omni](../qwen-omni/_index.md)。 diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index 6fce5e495..2ccedc1df 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5608"] +sources: ["PR #5157", "PR #5202", "PR #5608", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py] confidence: high --- @@ -18,7 +18,7 @@ confidence: high |---|---|---| | Qwen3-TTS、`qwen3_tts` pipeline | Q3TTS-1a/1b | `config/pipeline_registry.py::OMNI_PIPELINES["qwen3_tts"]`；`model_executor/models/qwen3_tts/pipeline.py` | | `ref_audio`、x-vector、ICL、artifact-only reuse | Q3TTS-1a/1b | `entrypoints/openai/serving_speech.py::_qwen3_tts_can_use_ref_audio_artifact_only`、`_track_ref_audio_artifact_warmup`、`_mark_ref_audio_artifact_ready_for_request` | -| talker/code2wav、deferred audio codes | Model Executor / Scheduler 共享规则 | `model_executor/models/qwen3_tts/qwen3_tts_talker.py::Qwen3TTSTalkerForConditionalGeneration`、`qwen3_tts_code2wav.py::Qwen3TTSCode2Wav` | +| talker/code2wav、delta frame、request cache、segmented graph | Q3TTS-3a/3b/3c/3d + Model Executor | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `qwen3_tts_code2wav.py::Qwen3TTSCode2Wav` → `segmented_graph_wrapper.py` | | OpenAI speech adapter | Q3TTS-1a/1b + Serving | `entrypoints/openai/tts_adapters/qwen3_tts.py::Qwen3TTSAdapter` → `serving_speech.py` | | NPU、RoPE、BNSD/BSND、`codec_chunk_ramp` | Q3TTS-2a | `platforms/npu/models/qwen3_tts_tokenizer_v2.py::_apply_rotary_pos_emb_npu` → `platforms/npu/layers/rotary_embedding.py::npu_rotary_mul_with_bsnd_fallback` | @@ -61,5 +61,89 @@ confidence: high parity。当前测试只把一个 unsupported shape 的 cos/sin dispatch 期望改为 BSND axis， 尚无专门的 supported/unsupported 边界矩阵或 eager 数值 parity。 ^[PR #5608] +## Q3TTS-3a — async connector 只在首块传 prefix，后续只传 delta + +- 触发：`async_chunk=true` 的 Talker→Code2Wav 分块、ICL reference code、 + x-vector-only 或 chunk ramp。 +- 强制：首块包含有界 ICL ref prefix 和已完成 codec frames；后续块只传 + 新完成 frames，`left_context_size=0`，依赖 Code2Wav 按 request 维护 + quantizer/conv/Transformer 状态。首个也是最后一个的短请求必须一次 + 发出全长；空的 finished payload 仍携带 request ID 以便清理。 +- 强制：async Code2Wav 缺 request ID 必须 fail fast；首个 ICL 状态若声明 + `ref_context_size` 却没有实际 prefix，也必须拒绝。非 async/stateless 路径 + 继续传全序列，不要求 request ID。 +- 验收：ICL/xvec、ramp/固定 initial chunk、首块即 final、空 EOF、丢失 + request ID/ref prefix，并证明后续 payload 没有重发旧 frames。 ^[PR #5202] + +## Q3TTS-3b — decoder state 必须以 scheduler ID 定位且在所有终止边界释放 + +- 强制：正常 runner 调用用 `request_ids` 中的 scheduler/internal ID 作 cache key， + 因为 `on_requests_finished()` 接收的是同一 ID；payload/external `meta.request_id` + 仅作 direct-forward 与 CUDA Graph dummy run 没有 runner ID 时的 fallback。 +- 强制：正常 `finished`、resumable `is_segment_finished`、空/无效最后载荷 + 和 scheduler abort 都要清理；segment 边界后不得继承上一段 KV/conv/ + quantizer 状态。dummy ID 需显式 `_is_dummy_run`；decoder dummy path 与正在 + 进行的外层 CUDA Graph capture 必须走 exact decode，且 dummy replay/fallback + 不得污染运行统计。 +- 边界：`_decoder_state_cache_warn_entries=512` 是 warning threshold，不是容量 + 上限；为保持 active request 正确性不做 LRU eviction。因此 ID mapping 与 + abort cleanup 是防止 GPU state 无界增长的必要合同，不能用“有 512 上限” + 掩盖泄漏。 +- 验收：internal/external ID 不同的 abort、finished/segment-finished、dummy direct call、 + 空/非法 row 并存的 batch。目标 pin 没有专门让 Code2Wav cache 在 + `is_segment_finished` 后清空的 focused test，该边界仍需补齐。 ^[PR #5202] + +## Q3TTS-3c — compacted batch 必须保持原 request slot，输出长度归 decoder + +- 强制：有状态请求按 mode（ICL/xvec）、phase（first/growing/rolling）、 + target suffix shape 分组，再恢复到原 request slot；一个 right-padded + `[B,Q,F]` 输入必须同时传真实 lengths 和对齐的 state list。跳过空/ + malformed row 后的 stateless leading-context trim 必须经 `valid_indices[row]` + 回到原 batch 位置，不得用 compacted row 直接索引 metadata。 +- 强制：decoder 返回每请求精确长度的 waveform list；stateless 分支移除 + right padding/内部 overlap，Code2Wav 只再移除 scheduler 声明的 leading ref + context。这一 stateless 合同也影响使用 `Qwen3TTSCode2Wav` 的 Aura-Omni， + 不能只测 Qwen3-TTS async 路径。 +- 验收：覆盖 empty/malformed row 与有效 ICL row 混合、variable prefix + left-padding mask、mixed-phase slot restoration，以及 stateless ref-audio leak/ + short-row tail 回归。^[PR #5202] + +## Q3TTS-3d — eager 与 CUDA Graph 必须共用可达 phase 合同 + +- 触发：修改 segmented graph、first/growing/rolling phase、chunk ramp、capture + shape/batch bucket 或 eager fallback。 +- 强制：无 ramp 时 phase 从 `initial_codec_chunk_frames`、`codec_chunk_frames`、 + decoder `sliding_window` 派生；有 ramp 时用实际 delta。async 忽略 + `decode_cudagraph_capture_sizes`；stateless 用默认 shape 与配置额外 shape 的 + **union**。`decode_cudagraph_batch_sizes` 只限 graph bucket，`decode_batch_max_size` + 只拆 eager stateful group（0 表示不限），两者不可混同。 + 旧 `decode_compile_shapes`/`decode_batch_bucket_frames` 已不再被 Qwen3 Code2Wav + 消费，不得将遗留 YAML 字段当作有效调优。 +- 降级：不支持的 phase/shape 或 graph 启用失败要 fallback eager 而不得 + 返空；ROCm/NPU/enforce-eager 都依赖 eager 合同，不能把 CUDA-only 测试 + 外推为跨平台验证。 +- 精度：增量解码与 full decode 数值接近但不保证 bit-exact，shape、 + layout 和 contiguity 均可引入小浮点差异。验收要覆盖 eager/graph、 + ICL/xvec、first/growing/rolling、partial-final trim、缺失 capture 与 graph-enable + 失败。^[PR #5202] + 最终 pin 没有 PR body 早期描述的 `_last_output_audio_length`；真实合同是 + exact-length `list[Tensor]` 加 Code2Wav 的 stateless leading-context trim。 + +## PR #5202 性能证据边界 + +- 作者基准是单张 RTX 3090 24 GB、vLLM 0.26.0、Omni + `b8edbc8d`、Seed-TTS 32 prompts、concurrency 1/2/4/8/16、batch-size-1 graph + capture；cached RTF 从 `0.222` 至 `0.718`，相对 baseline 报告吞吐改善 + `10.3%/20.2%/31.1%/40.4%/34.1%`，mean WER `0.0089`。这是特定 + 配置与数据集证据，无 raw result 链接/重复次数或方差，不可当作 + 通用 SLA。PR 自报 focused tests `34+22+101 passed`，同时保留一个 + 未改动的 log-capture 失败，不得记为全绿。 + 该 `b8edbc8d` 是 pre-merge 基准 SHA，不是目标 `4f9b6507` 的性能证明。 +- 最终 reviewer 在 H20、vLLM 0.26.0 上用 current main `12a5f6fb` + 对照 head `c63c2efc`，8.08 s clone reference 的输出前缀相关系数为 + base `-0.0034/+0.0028`、head `+0.0005/-0.0038`，支持“未重现 + reference-audio leak”。该单请求 A/B 不覆盖 compacted-batch index fix， + 它的证据来自新单元测试，不得混为硬件验证。 + 共享 readiness/错误隔离规则见 [Serving rules](../../components/serving/rules.md)； Qwen 家族入口见 [Qwen-Omni](../qwen-omni/_index.md)。 From 21d580869a6ce1e1e2ddaddfb30271274f2e862f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 00:19:29 +0800 Subject: [PATCH 090/465] knowledge(vllm-omni): sync 1f65814f (PR #5978) Upstream-Commit: 1f65814fab903abedea264369a7cc1b591452ba0 Upstream-PR: #5978 Knowledge-Delta: MiniMax H3 media-window and accuracy contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 4 +- .../diffusion/rules-output-lifecycle.md | 18 ++++- .../vllm-omni/components/diffusion/rules.md | 4 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +- .../vllm-omni/models/minimax-h3/_index.md | 17 +++-- .../models/minimax-h3/rules-media.md | 76 +++++++++++++++++++ .../vllm-omni/models/minimax-h3/rules.md | 56 +------------- 14 files changed, 118 insertions(+), 79 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8b89e1861..409e78b8a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 12a5f6fbf3ac08ffeec8cb4a2fc05344b8d1e4cf + previous_audited_sha: 4f9b65074dead2e89f8ace4798a92f2370aec85b audited_ref: main - audited_sha: 4f9b65074dead2e89f8ace4798a92f2370aec85b + audited_sha: 1f65814fab903abedea264369a7cc1b591452ba0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 169bd8c82..98042dd69 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4f9b65074dead2e89f8ace4798a92f2370aec85b`. The canonical machine baseline +1f65814fab903abedea264369a7cc1b591452ba0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bf5982fc1..ecff2053a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4f9b6507` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1f65814f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 70378f399..c905d3fcb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4f9b6507` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1f65814f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -32,7 +32,7 @@ sources: [] | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | -| DLO DP collective wave、per-worker result queue、async pump、SHM ownership 与 shutdown | [multiprocess runtime rules](rules-output-lifecycle.md) | +| video/audio mux、DLO DP wave、result queue、async pump、SHM ownership 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | request-wave admission coalescing、stable window、deadline 与 finite config | [admission wait rules](rules-admission-wait.md) | | Scheduler-managed diffusion KV 的请求控制面、Hunyuan layout 与未实现边界 | [paged KV control plane](paged-kv-control-plane.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 7705a7c59..d7ce9ed2e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -1,14 +1,14 @@ --- -title: "Diffusion multiprocess runtime 规则" +title: "Diffusion output 与 multiprocess runtime 规则" created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5864", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py] +sources: ["PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- -# Diffusion multiprocess runtime 规则 +# Diffusion output 与 multiprocess runtime 规则 ## DIFF-1d — async output 的 compute 与 output-ready 必须分成两个可观察阶段 @@ -68,3 +68,15 @@ confidence: high 2×B300 evidence 覆盖 ordinary TP2 与 H3 DLO DP2 的错误 wave→同 engine recovery→无 active child； 仍只绑定 eager+CUDNN exact topology，不能推广到一般 DP/SP 或替代 per-replica SP/mmap redesign。 ^[PR #5864] + +## DIFF-1i — AAC mux 必须显式标记输入时间零点 + +- 触发：修改共享 mux 的 audio-frame timestamp、codec/container 或 video API 输出。 +- 强制：共享 `mux_video_audio_bytes()` 在构造 AAC frame 时设 `pts=0`、 + `time_base=1/sample_rate`，让 MP4 以负 priming timestamp 表达 encoder delay， + 避免把 delay 暴露成前导静音。这是共享 mux caller 的 blast radius，不是 + MiniMax H3 专属。 +- 禁止：从负 AAC packet PTS 外推其他 codec/container、audio sample/duration/ + content、唇音同步或完整 A/V alignment。 +- 验收：目标 focused test 解 mux 后断言第一个有 PTS 的 AAC packet `PTS<0`； + 样本和 A/V 对齐结论仍需独立的端到端波形/时序验收。^[PR #5978] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 071ddccd5..69ac75b2f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -20,7 +20,7 @@ confidence: high | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| -| CUDA Graph、compile、fused solver/norm/RoPE、FA determinism、eager parity、tensor dtype/device、async output/teardown | `execution-parity`：`DIFF-1a`–`1h` | `compile.py::regionally_compile` → shared layer/backend → model denoise/output pump/shutdown | +| CUDA Graph、compile、fused solver/norm/RoPE、FA determinism、eager parity、tensor dtype/device、async output/teardown | `execution-parity`：`DIFF-1a`–`1i` | `compile.py::regionally_compile` → shared layer/backend → model denoise/output pump/shutdown | | seed、request-local generator、guidance=0、并发 RNG、batched generators | `execution-parity`：`DIFF-1b` | `inputs/data.py::OmniDiffusionSamplingParams` → runner `_initialize_generator` → request-batch generator collate | | ModelOpt/checkpoint adapter、weight/scale remap、unknown tensor、resolution path | `checkpoint-distributed`：`DIFF-2a` | `diffusers_loader.py::{_get_checkpoint_adapter,load_weights}` → `modelopt.py::{_resolve_target_and_output_names,adapt}` | | host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | @@ -36,7 +36,7 @@ confidence: high | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | -| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1h` | +| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1i` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2k` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4i` | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ba256a55a..2cb846df6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4f9b6507` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1f65814f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index d2dc9368c..a2083eaf8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4f9b6507` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1f65814f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a18585111..da85dff2d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4f9b6507` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1f65814f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7d77443f5..15d6567ef 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4f9b6507` 验证存在 +- 源码校验：以上路径均已在 `main @ 1f65814f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c7620b50b..16b6ad63a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4f9b6507`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1f65814f`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 12a5f6fb \ - --to 4f9b6507 \ + --from 4f9b6507 \ + --to 1f65814f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 8c5020e08..6c728aab8 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -90,12 +90,13 @@ Hub/local model lookup 合同由 [ComfyUI tooling rules](../../tooling/rules.md) ## 验证入口 模型专属 contract、packing 和 parallel 测试在 `tests/diffusion/models/minimax_h3/`。 -T2VA full-model accuracy 入口在 `tests/e2e/accuracy/minimax_h3/`：模型 snapshot 固定在 -`73372e6c`，但该 revision 没有参考视频，official `assets/t2va.mp4` 实际从 Hugging Face -`resolve/main` 下载，不能把 golden 写成 revision-immutable。用例固定 1344x768/24 FPS/ -243 帧/50 steps/seed 0，除视频和 AAC 32 kHz stereo metadata 外，还以 SSIM >= 0.82、 -PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、text-encoder TP4 -和 VAE patch parallel 4。该用例是精度/媒体合同，不是性能基线。 +full-model accuracy/nightly 入口现保留 I2VA 与 Ref2VA，两者都从 +Hugging Face `main` 下载模型分区与 SHA-256 固定的 official assets；模型 +`main` 仍可漂移。两例固定 1344x768/24 FPS/AAC 32 kHz stereo/50 steps/ +seed 0，帧数分别 192/124，共用 SSIM>=0.97、PSNR>=34 dB gate。nightly +lane 声明 4x H100，PR 数值则来自 4x B300，不得混为同一硬件证据。 +T2VA 因当前严格 gate 未达标被移出；这表示“无 retained pixel-level gate”， +不是 T2VA 功能不支持。该用例是精度/媒体合同，不是性能基线。^[PR #5978] 硬件 recipe 只记录已验证的 GPU/NPU 形状；性能数字不能从 recipe 的配置示例泛化为全硬件 保证。目标 pin 的 MUSA Ref2VA recipe 已改为 `MODEL_ROOT/Ref2VA` + `--task-type ref2va`； 它只验证单分区启动，不能外推 combined 双 DiT 的容量或性能。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 @@ -104,6 +105,6 @@ PSNR >= 20 dB gate 完整输出；nightly lane 使用 4x H100、USP4、HSDP4、t ## 审查入口 H3 input matrix/media ingress，以及 text-encoder completeness、online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 -[MiniMax H3 rules](rules.md#direct-代码快速入口)；checkpoint transform、quantized loader 与 +[media rules](rules-media.md) 与 [MiniMax H3 rules](rules.md#direct-代码快速入口)；checkpoint transform、quantized loader 与 text-encoder fused-source 完整性正文见 [loading rules](rules-loading.md)；DLO、consumer/H100/ROCm 部署和硬件证据正文见 [deployment rules](rules-deployment.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md new file mode 100644 index 000000000..e654d809a --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md @@ -0,0 +1,76 @@ +--- +title: "MiniMax H3 媒体输入与精度规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5752", "PR #5829", "PR #5978", .buildkite/cuda/test-nightly.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/entrypoints/openai_api/test_video_server.py] +confidence: high +--- + +# MiniMax H3 媒体输入与精度规则 + +## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 + +- 触发：H3 `t2va`/`fl2va`/`ref2va`、reference 计数、frame index、canvas、duration 或 + `num_outputs_per_prompt`。 +- 强制：FL2VA 接受一张首/尾帧或两张有序首尾帧，signature 仅为 `[0]`、`[-1]`、 + `[0,-1]`，未显式给出时一张默认 `[0]`、两张默认 `[0,-1]`。Ref2VA 必须至少有一项 + image/video visual reference；image≤9、video≤3、standalone audio≤3、总数≤12，允许 + image-only 及 image/video/audio mixed matrix，但拒绝 audio-only。 +- 强制：输出固定 24 FPS、32 kHz audio、duration 4–15 秒。省略 width/height 时 + `short_edge` 只能为 768；T2VA 即使同时给出 width/height，仍必须显式选择 `21:9`、 + `16:9`、`4:3`、`1:1`、`3:4`、`9:16`，因为 named ratio 在 explicit-dimension branch + 之前解析。FL2VA 跟随首图 geometry 并忽略通用 ratio override，Ref2VA 默认 `16:9`， + `adaptive`/`auto` 是该默认的 alias。^[PR #5829] +- 强制：`num_outputs_per_prompt` 仅为 1–10，各输出 seed 是 `seed + output_index`；pipeline + tensor output 与未被公开 caller 使用的 `generate_videos()` 保留 fan-out。此 pin 上 public + async job 与 `/v1/videos/sync` 都调用 `generate_video_bytes()`，多输出时告警并只持久化/返回 + `artifacts.videos[0]`；不得把 recipe 的“async 返回全部”描述当成实现证据。 +- 强制：Ref2VA video frames 和 embedded/standalone audio conditions 都以生成时域为上界： + ffmpeg 用 `target_frame_count`；embedded audio 用 + `min(source_segment_duration,target_frame_count/24)`；standalone audio 用 + `num_frames/(sampling.fps or 24)`。source segment `duration_seconds` 仍单独保留作 + admission/metadata，不得被 bounded conditioning 覆盖。 +- 禁止：把 partition 名当输入能力；FL2VA/Ref2VA 权重分区仍需与 task 匹配。 + `max_duration_seconds` 先转 float 并在任何 rank branch/collective 前拒绝 `<=0`； + 否则 rank 0 raise 可使其他 rank 卡在 broadcast。实现没有 `isfinite` 检查， + NaN/+inf 可越过前置 guard 并在 rank 0 后续失败；这是多 rank 验收缺口。 +- 验收：以 F1/F2、R1–R10、G1–G4 matrix 覆盖 mixed reference、audio-only、 + ratio/default、duration、fan-out/seed/public first-only；另覆盖 video/embedded/standalone audio + 的生成时域上界、frame-count/FPS 一致性与非零 rank 非正 duration 在 collective + 前 fail-fast。^[PR #5752] ^[PR #5978] + +## MMH3-2b — media ingress 在解码前限界，request 错误保持 HTTP 400 + +- 触发：typed `image_reference`/`video_reference`/`audio_reference`、multipart + `input_reference(s)`、URL/data URL 下载、ffprobe/transcode 或 `start_time_seconds`。 +- 强制：typed 字段接受单对象或有序 list；multipart 按 MIME/suffix 分类并保持每种媒体顺序。 + 仅 H3 multipart 在读取/解码前 bounded read：image≤30 MiB（JPEG/PNG/WEBP/HEIC/HEIF）、 + video≤50 MiB（MP4/MOV，H.264/H.265，audio 若有则 AAC/MP3）、audio≤15 MiB + （WAV/MP3，PCM/MP3）。image/video dimension 256–5760、ratio 0.4–2.5、video FPS 23.976–60。 +- 强制：每个 video/audio 2–15 s，video 总时长≤15 s，embedded+standalone audio + conditioning 总时长≤15 s。`start_time_seconds` 在 source 内且剩余至少 2 s； + 只容忍官方双视频 container probe 的微小 rounding overflow 并裁到剩余时长， + 真实 overflow 仍拒绝。 +- 强制：typed URL/data-URL video materialize 为 request-scoped MP4，成功/异常都清理； + path-backed media 下载后由 model preparation 检 metadata。mixed image+video 只在 model + metadata 声明 capability 时放行；非零 DiT rank 用已广播 `video_count` 判定 visual。 +- reference transcode 用 `libx264rgb -crf 0 -pix_fmt rgb24`，让 Qwen3-VL 抽样和 + video VAE 消费同一 prepared RGB stream。sampling 对该 stream 只 decode 一次； + 测试必须同时断言完整 sampled arrays、block timestamps 和单次 decode， + 不得只测 call count。这证明相同 prepared pixels，不证明任意 decoder 的 + byte identity。^[PR #5978] +- 禁止：PIL/ffmpeg 转换后才检原格式/大小；让 request-facing `ValueError` 变 HTTP 500； + 用 fake engine 成功代替 source-path、cleanup/capability 证明。验收要覆盖非法计数、 + multipart 限界、path metadata/segment、typed URL/data URL、mixed、unsupported model、temp cleanup + 和多 rank video+audio，均在生成前给 `OmniClientError`/HTTP 400。typed URL/data URL + 仍先完整 download/base64 decode，typed image 仍丢原 size/format，故 multipart bounded + 证据不可外推；H3 API helper 仍有泛化重构债务。^[PR #5752] +- retained nightly 只有 I2VA（192 frames/8 s）和 Ref2VA（124 frames/5 s），共用 + 1344×768@24 FPS、50 steps、seed 0、SSIM≥0.97/PSNR≥34 dB。这是 video-pixel + similarity + AAC 32 kHz stereo container metadata gate，不比 audio samples/duration/content、 + lip sync 或 A/V alignment。assets 有 SHA-256，模型 revision 仍是 mutable `main`。 + nightly 声明 4×H100；PR 的 I2VA 0.975190/35.981286、Ref2VA 0.980577/ + 42.650204 来自 4×B300 外部运行，不是 checked-in 结果。T2VA 因严格 gate + 未达标被删除，仅表示无 retained pixel-level gate，不表示功能不支持。^[PR #5978] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index bbbcacd8f..97b8aadb8 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5752", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/inputs/data.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/entrypoints/openai_api/test_video_server.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] confidence: high --- @@ -24,8 +24,8 @@ confidence: high | TRTLLM、ragged packed metadata、SAGE、Skip-Softmax、Blackwell default | `MMH3-1e` | H3 metadata/roles → TRTLLM packed trim/quant gate → platform default | | text encoder、missing q/k/v 或 gate/up、eager load bookkeeping | [MMH3-1f](rules-loading.md#mmh3-1f-text-encoder-eager-load-必须证明每个-source-shard-完整) | `encoder.py::_load_weights` → `pipeline_minimax_h3.py::load_weights` strict report | | NPU packed varlen、quadratic mask、LaserAttention、prefix K/V | `MMH3-1g` | H3 packed producer → backend capability/metadata → NPU FlashAttention fallback | -| FL2VA keyframe、Ref2VA mixed reference、shape/output matrix | `MMH3-2a` | `pipeline_minimax_h3.py::{_resolve_fl2va_keyframe_indices,_validate_ref2va_reference_counts,_resolve_shape}` | -| media limit、typed/multipart reference、HTTP 400、temp source | `MMH3-2b` | `api_server.py` video handlers → `serving_video.py::_run_and_extract` → `reference_video.py` | +| FL2VA keyframe、Ref2VA mixed reference/时域限界、shape/output matrix | [MMH3-2a](rules-media.md#mmh3-2a-taskreferenceshape-与多输出必须作为一个输入矩阵维护) | `pipeline_minimax_h3.py` → `reference_video.py` | +| media limit、typed/multipart reference、HTTP 400、temp source | [MMH3-2b](rules-media.md#mmh3-2b-media-ingress-在解码前限界request-错误保持-http-400) | `api_server.py` → `serving_video.py` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | | modular checkpoint、combined/partition task、两套 DiT、shared component | `MMH3-2d` | model index discovery → startup task selection → task-specific transformer/cache lifecycle | | request `quality`、lossless/high、dynamic Cache-DiT | `MMH3-2e` | request sampling → quality policy → request Cache-DiT runtime → denoise | @@ -165,56 +165,6 @@ confidence: high 替代；同 seed E2E output comparison 与重新抓 memory snapshot 在 PR 中仍列为 follow-up。 ^[PR #5891] ^[PR #5997] -## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 - -- 触发：H3 `t2va`/`fl2va`/`ref2va`、reference 计数、frame index、canvas、duration 或 - `num_outputs_per_prompt`。 -- 强制：FL2VA 接受一张首/尾帧或两张有序首尾帧，signature 仅为 `[0]`、`[-1]`、 - `[0,-1]`，未显式给出时一张默认 `[0]`、两张默认 `[0,-1]`。Ref2VA 必须至少有一项 - image/video visual reference；image≤9、video≤3、standalone audio≤3、总数≤12，允许 - image-only 及 image/video/audio mixed matrix，但拒绝 audio-only。 -- 强制：输出固定 24 FPS、32 kHz audio、duration 4–15 秒。省略 width/height 时 - `short_edge` 只能为 768；T2VA 即使同时给出 width/height，仍必须显式选择 `21:9`、 - `16:9`、`4:3`、`1:1`、`3:4`、`9:16`，因为 named ratio 在 explicit-dimension branch - 之前解析。FL2VA 跟随首图 geometry 并忽略通用 ratio override，Ref2VA 默认 `16:9`， - `adaptive`/`auto` 是该默认的 alias。^[PR #5829] -- 强制：`num_outputs_per_prompt` 仅为 1–10，各输出 seed 是 `seed + output_index`；pipeline - tensor output 与未被公开 caller 使用的 `generate_videos()` 保留 fan-out。此 pin 上 public - async job 与 `/v1/videos/sync` 都调用 `generate_video_bytes()`，多输出时告警并只持久化/返回 - `artifacts.videos[0]`；不得把 recipe 的“async 返回全部”描述当成实现证据。 -- 禁止：把 partition 名当输入能力；FL2VA/Ref2VA 权重分区仍需与 task 匹配。不得把长 - reference clip 截成输出长度；reference 自己的 segment/duration 合同由 MMH3-2b 约束。 -- 验收：以 F1/F2、R1–R10、G1–G4 matrix 覆盖正反例，包含 mixed reference、audio-only、 - ratio/default、duration 边界、pipeline fan-out、seed 顺序与两个 public endpoint 的 first-only - 行为；文档与实现冲突必须显式暴露。 ^[PR #5752] - -## MMH3-2b — media ingress 在解码前限界，request 错误保持 HTTP 400 - -- 触发：typed `image_reference`/`video_reference`/`audio_reference`、multipart - `input_reference(s)`、URL/data URL 下载、ffprobe/transcode 或 `start_time_seconds`。 -- 强制：typed 字段接受单对象或有序 list；multipart 按 MIME/suffix 分类并保持每种媒体的 - 顺序。仅 H3 multipart `input_reference(s)` 在读取/解码前执行 bounded read 与 allowlist： - image≤30 MiB（JPEG/PNG/WEBP/HEIC/HEIF）、 - video≤50 MiB（MP4/MOV，H.264/H.265，若有 audio 则 AAC/MP3）、audio≤15 MiB（WAV/MP3， - PCM/MP3）。image/video dimension 为 256–5760、ratio 为 0.4–2.5；video FPS 23.976–60。 -- 强制：每个 video/audio reference 时长 2–15 秒；video 总时长≤15 秒，embedded video - audio 与 standalone audio 合并后的 conditioning 总时长也≤15 秒。每个 video 的 - `start_time_seconds` 必须落在 source duration 内并剩余至少 2 秒。只容忍官方双视频 - container probe 的微小 rounding overflow，并裁到剩余总时长；真实 overflow 仍拒绝。 -- 强制：typed URL/data-URL video 必须 materialize 为 request-scoped 临时 MP4 source，再交给 - 只收 path 的 preparation，成功/异常都清理；path-backed typed video/audio 在下载后由模型 - preparation 检查 file metadata。mixed image+video 只有 model metadata 明示 capability 才放行； - 分布式非零 DiT rank 用已广播的全局 `video_count` 判断 visual 是否存在。 -- 禁止：PIL/ffmpeg 转换后才检查原始格式/大小；把 request-facing `ValueError` 变成 HTTP 500； - 用 fake engine 的成功代替 source-path、cleanup 或 cross-model capability 证明。 -- 验收：非法计数、multipart 格式/大小及 path-backed media metadata/segment 均在生成前返回 - `OmniClientError`/HTTP 400；覆盖 typed URL/data URL、multipart mixed、unsupported model、 - temp cleanup 和多 rank video+audio。此 pin 的 typed URL/data URL helper 仍先完整 - download/base64 decode；typed image - 还会丢失原始 size/format，因此 multipart 的 bounded/predecode 证明不可外推，属于验证与 - hardening 缺口。API server 中的 H3 专用 helper 也仍有 reviewer 指出的后续泛化重构债务。 - ^[PR #5752] - ## MMH3-2c — conditioned VAE 的固定种子必须按实际设备隔离并恢复 - 触发：修改 H3 image/video reference 的 VAE encode、固定 keyframe seed、`fork_rng`、设备 From 52e6a53d79fd97fd2a0a9cb08378d64dc6fef25f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 00:34:58 +0800 Subject: [PATCH 091/465] knowledge(vllm-omni): sync 1a56953d (PR #5991) Upstream-Commit: 1a56953d7ac271d6a5afb9313e3c3f741f9e1a9a Upstream-PR: #5991 Knowledge-Delta: document distilled sigma schedules and H3 partition isolation --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../vllm-omni/components/diffusion/rules.md | 4 +-- .../components/diffusion/sigma-schedules.md | 29 +++++++++++++++++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/minimax-h3/_index.md | 4 ++- .../vllm-omni/models/minimax-h3/rules.md | 32 ++++++++++++++++++- 13 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/sigma-schedules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 409e78b8a..f7a1121cd 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4f9b65074dead2e89f8ace4798a92f2370aec85b + previous_audited_sha: 1f65814fab903abedea264369a7cc1b591452ba0 audited_ref: main - audited_sha: 1f65814fab903abedea264369a7cc1b591452ba0 + audited_sha: 1a56953d7ac271d6a5afb9313e3c3f741f9e1a9a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 98042dd69..4804abb22 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1f65814fab903abedea264369a7cc1b591452ba0`. The canonical machine baseline +1a56953d7ac271d6a5afb9313e3c3f741f9e1a9a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ecff2053a..1b35555af 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1f65814f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1a56953d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c905d3fcb..ac413e269 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1f65814f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1a56953d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -33,6 +33,7 @@ sources: [] | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | | video/audio mux、DLO DP wave、result queue、async pump、SHM ownership 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | +| distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | request-wave admission coalescing、stable window、deadline 与 finite config | [admission wait rules](rules-admission-wait.md) | | Scheduler-managed diffusion KV 的请求控制面、Hunyuan layout 与未实现边界 | [paged KV control plane](paged-kv-control-plane.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 69ac75b2f..89570d046 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -30,7 +30,7 @@ confidence: high | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | | modular/multi-DiT、`_dit_modules`、request-scoped Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2j` | request batch key → model policy/cache runtime；pipeline component list → shared consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | -| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4i` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | +| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4j` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | | 审查组 | 什么时候触发 | 规则 ID | @@ -39,7 +39,7 @@ confidence: high | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1i` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2k` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4i` | +| `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | ## 优化路径与 eager 的等价合同 diff --git a/knowledge/repos/vllm-omni/components/diffusion/sigma-schedules.md b/knowledge/repos/vllm-omni/components/diffusion/sigma-schedules.md new file mode 100644 index 000000000..6925fce01 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/sigma-schedules.md @@ -0,0 +1,29 @@ +--- +title: "Diffusion distilled sigma schedule 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion, scheduler] +sources: ["PR #5991", vllm_omni/diffusion/models/dmd2/config.py, vllm_omni/diffusion/models/dmd2/mixin.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py] +confidence: high +--- + +# Diffusion distilled sigma schedule 规则 + +## DIFF-4j — continuous sigma boundaries 不得与 scheduler integer timesteps 混同 + +- 触发：新增 distilled/DMD2 checkpoint schedule metadata、变更 sigma shift 或把 + `DMD2SigmaSchedule` 接入新 pipeline。 +- 强制：continuous `base_schedule` 至少两个有限数，严格从 1.0 递减到 + 0.0；先拒绝非有限值，再逐对验证相邻位置严格单调。区间数 + `len(boundaries)-1` 才是 request/cache/denoise step count，solver 保留全 boundary list。 +- 强制：`from_metadata()` 的 absent key 与 explicit null 都因 `metadata.get()` 返回 + `None` 而保留 legacy schedule；explicit empty list 是 malformed，不得静默 fallback。 + 一个 base schedule 可独立应用多个 modality shift，但不等于 + `DMD2Config.denoising_timesteps` 的 scheduler integer timesteps。 +- 禁止：把共享 utility 存在解读为 `DMD2PipelineMixin` 已接入。目标 pin + 中 mixin 仍替换 `DMD2EulerScheduler`，未消费该 continuous schedule class；新接入者 + 必须自行定义 schedule resolution 与 request sanitization 合同。 +- 验收：覆盖长度、端点、单调、NaN/inf positions、absent/null/empty metadata、 + interval count 及不同 shift 值。`shifted_sigmas()` 目标实现只在 float 转换前检 + `shift_scale<=0`；NaN/+inf 可通过并产生非有限输出，尚缺有限性验收。^[PR #5991] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2cb846df6..4c14ec05f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1f65814f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1a56953d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a2083eaf8..28fe7e4f7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1f65814f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1a56953d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index da85dff2d..b52eb677d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1f65814f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1a56953d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 15d6567ef..0ba24ceed 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1f65814f` 验证存在 +- 源码校验：以上路径均已在 `main @ 1a56953d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 16b6ad63a..11419a137 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1f65814f`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1a56953d`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 4f9b6507 \ - --to 1f65814f \ + --from 1f65814f \ + --to 1a56953d \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 6c728aab8..f4c82c343 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -70,6 +70,8 @@ confidence: high mutable image 和 support-table 不一致边界见 MMH3-3d；不得从 gfx architecture 名直接扩展 SKU。 - TeaCache 只 hook FL2VA DiT，Ref2VA-only 拒绝；model default、public 入口覆盖、request `quality` 交互与单 H100 校准边界见 MMH3-2g。 +- distilled checkpoint 的 continuous RF `base_schedule`、四步语义与 partition + 隔离见 MMH3-2h；它不替换 H3 的自定义 solver。 - datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；packed suffix trim、SAGE 短序列 role、 mask/metadata fail-closed、AllGather-KV 禁用及性能证据边界见 MMH3-1e。 - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 97b8aadb8..d1868f7b2 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] confidence: high --- @@ -31,6 +31,7 @@ confidence: high | request `quality`、lossless/high、dynamic Cache-DiT | `MMH3-2e` | request sampling → quality policy → request Cache-DiT runtime → denoise | | force-refresh hint、once/repeat、reinstall key | `MMH3-2f` | H3 `extra_args` validation → immutable cache config → installation key/refresh context | | TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | `MMH3-2g` | custom enabler → H3 extractor → module-resident hook state / per-generation reset | +| distilled、DMD2、`base_schedule`、4-step、sigma boundary | `MMH3-2h` | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | | DLO、TP-local、resident layers、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | @@ -275,5 +276,34 @@ confidence: high 证明。单 H100 online/offline、 Cache-DiT default/conservative 数据无多 prompt/repeats，不能升级为 gate。 +## MMH3-2h — distilled sigma schedule 按 partition 所有且以区间计步 + +- 触发：H3 partition `model_index.json` 的 `_minimax_h3.base_schedule`、显式 + `num_inference_steps`、video/audio flow shift 或 combined FL2VA+Ref2VA。 +- 强制：缺少 metadata key 或显式 null 都表示未蒸馏，保持旧 + `num_inference_steps or 50` 的 uniform-point 构造（默认 50 个点实际是 49 个 solver + intervals）；explicit empty list 必须拒绝。调度至少两个有限点，严格 + 从 1.0 递减到 0.0，并逐对验证相邻位置。 +- 强制：5 个 sigma boundaries 只有 4 个 denoise intervals；`num_inference_steps`、 + Cache-DiT quality policy 和请求验证均使用 `len(base_schedule)-1`，solver 仍接收 + 完整 boundary list。用户省略 step 时自动采用 checkpoint 步数；显式值只有 + 精确相等才接受，否则 `OmniClientError`。同一 continuous base positions 分别 + 应用 video/audio shift（默认 12/3），不得改用单模态 integer timesteps。 +- 强制：schedule 从各 partition metadata 分别读取；`t2va`/`fl2va` 选 FL2VA， + `ref2va` 选 Ref2VA。distilled FL2VA 不得拖普通 Ref2VA 进 4-step。类级空 map + 是 partially constructed pipeline 的 legacy fallback，避免 `object.__new__`/dummy fixture + 读未初始化属性。 +- 边界：共享 `DMD2SigmaSchedule` 在 `diffusion/sched` 定义，但此 pin 只有 + H3 消费；现有 `DMD2PipelineMixin` 仍是 scheduler-backed，用 + `DMD2EulerScheduler`/integer `denoising_timesteps`，本次未接入新 utility。不得由 + `[NPU]` title 推导 NPU 实机已验：无 platform/deploy/recipe 改动，PR hardware + 与 branch-head 也留空；附件视频只是 smoke，无数值 teacher/student quality gate。 +- 共享 boundary/interval、metadata null/empty 与 shift 有限性合同见 + [DIFF-4j](../../components/diffusion/sigma-schedules.md#diff-4j-continuous-sigma-boundaries-不得与-scheduler-integer-timesteps-混同)。 +- 验收：shared class 覆盖长度/端点/单调/非有限/shift≤0、缺 key 与空 key； + H3 覆盖双 shift 精确值、combined partition 隔离、4-step 传入 solver/quality、 + matching/mismatched explicit step 和 legacy/partially-constructed fallback。仍缺真实 distilled + checkpoint 的 CUDA/NPU E2E 与 teacher/student 质量阈值。^[PR #5991] + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 From d4c308ec1ae91e637f1b38fb98c821ad1ad07ea9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 00:44:17 +0800 Subject: [PATCH 092/465] knowledge(vllm-omni): sync 73a312d5 (PR #6031) Upstream-Commit: 73a312d5b6fe959318fd13dcd734f7b817b174fa Upstream-PR: #6031 Knowledge-Delta: document offline NumPy image save normalization --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- knowledge/repos/vllm-omni/tooling/_index.md | 4 ++-- knowledge/repos/vllm-omni/tooling/rules.md | 23 ++++++++++++++++++- 11 files changed, 36 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f7a1121cd..423194b8c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1f65814fab903abedea264369a7cc1b591452ba0 + previous_audited_sha: 1a56953d7ac271d6a5afb9313e3c3f741f9e1a9a audited_ref: main - audited_sha: 1a56953d7ac271d6a5afb9313e3c3f741f9e1a9a + audited_sha: 73a312d5b6fe959318fd13dcd734f7b817b174fa # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4804abb22..c8d9c03e9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1a56953d7ac271d6a5afb9313e3c3f741f9e1a9a`. The canonical machine baseline +73a312d5b6fe959318fd13dcd734f7b817b174fa`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1b35555af..4c27b48a6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1a56953d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 73a312d5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ac413e269..3241fc1af 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1a56953d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 73a312d5` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4c14ec05f..9da0c2e32 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1a56953d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 73a312d5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 28fe7e4f7..b74312661 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1a56953d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 73a312d5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b52eb677d..febb4f9af 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1a56953d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 73a312d5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0ba24ceed..5f2c2cc22 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1a56953d` 验证存在 +- 源码校验：以上路径均已在 `main @ 73a312d5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 11419a137..13bb7e5e6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1a56953d`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 73a312d5`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 1f65814f \ - --to 1a56953d \ + --from 1a56953d \ + --to 73a312d5 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/tooling/_index.md b/knowledge/repos/vllm-omni/tooling/_index.md index 9390ee354..016fe9fd8 100644 --- a/knowledge/repos/vllm-omni/tooling/_index.md +++ b/knowledge/repos/vllm-omni/tooling/_index.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: index tags: [vllm-omni] -sources: [apps/ComfyUI-vLLM-Omni/] +sources: [apps/ComfyUI-vLLM-Omni/, examples/offline_inference/text_to_image/text_to_image.py] --- # Apps 与 tooling @@ -14,4 +14,4 @@ sources: [apps/ComfyUI-vLLM-Omni/] | 遇到什么 | 查看哪里 | |---|---| -| ComfyUI node 输入、multipart payload、model spec 或参数 builder | [tooling rules](rules.md) | +| ComfyUI 输入/参数或离线 text-to-image artifact 保存 | [tooling rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/tooling/rules.md b/knowledge/repos/vllm-omni/tooling/rules.md index 5eeec929a..eea20d17b 100644 --- a/knowledge/repos/vllm-omni/tooling/rules.md +++ b/knowledge/repos/vllm-omni/tooling/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni] -sources: ["PR #5756", apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py, tests/e2e/features/comfyui/test_comfyui_integration.py] +sources: ["PR #5756", "PR #6031", apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py, examples/offline_inference/text_to_image/text_to_image.py, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/examples/offline_inference/test_text_to_image.py] confidence: high --- @@ -18,6 +18,7 @@ confidence: high |---|---|---| | ComfyUI video node、frame/reference、multipart payload | `OMNI-TOOL-1a` | `nodes.py` → `utils/types.py` → `utils/api_client.py` | | ComfyUI model path、partition、params builder | `OMNI-TOOL-1b` | `utils/models.py` → `utils/api_client.py` → integration test | +| offline T2I NumPy output、PIL、artifact save | `OMNI-TOOL-2a` | `text_to_image.py::{_normalize_images_for_save,main}` → README example test | ## OMNI-TOOL-1a — reference 输入状态与 multipart 字段必须一起验证 @@ -41,3 +42,23 @@ confidence: high - 验收：Hub ID、本地绝对路径及两个 partition 都命中同一 model spec，并断言 builder 收到 正确 keyword 和最终字段层级。PR review 曾直接发现 basename lookup 与错误 builder keyword， 因此这两项必须作为回归 fence。 ^[PR #5756] + +## OMNI-TOOL-2a — NumPy image 只在离线示例的 artifact 边界转为 PIL + +- 触发：修改共享 text-to-image 示例的 output selection、fallback extraction、输出类型或最终 + artifact 保存。 +- 强制：示例先按既有优先级选择非空 `output.images`、`request_output.images`，否则调用共享 + extractor；确认结果非空后，才逐个把直接 `np.ndarray` 元素经 Diffusers `numpy_to_pil` + 转换并展平返回的 PIL list，既有非 ndarray 元素原样保留，随后才逐图 `.save()`。这是离线 + artifact adapter，不改变 producer、production serving 或共享 `extract_images_from_outputs()` + 合同；当前已证明的 NumPy producer 是 LingBot T2I 的非空 `list[np.ndarray]`。 +- 禁止：把非空 NumPy list 直接当作 PIL 保存；为修示例而全局强制 production output 为 PIL； + 把此 helper 推广为任意 array/tensor/object 的通用 coercion。bare ndarray 在现有 selection 的 + truthiness 判断仍可能报 ambiguous truth value；direct tensor 与其他不支持 `.save()` 的对象会 + 原样通过 normalization，仍可在保存处失败。 +- 验收：至少覆盖 `[0,1]` HWC float ndarray 转换后的可保存性、尺寸/mode，以及 PIL passthrough； + mixed list、batched ndarray、非法 shape/dtype、bare ndarray 和 direct tensor 应作为明确回归矩阵。 + PR #6031 的单张 L20X 前后对照用 checksum 固定的 LingBot 1.3B revision，证明相同请求由保存处 + `AttributeError` 变为 320×192 RGB PNG；它是有界的 LingBot E2E artifact 证据，不证明共享 + utility、在线 serving 或其他 producer。目标提交没有新增 helper 自动化测试，README full-model + case 也不能替代上述类型矩阵。^[PR #6031] From 7e391776fbb575a894ce6b4b3ef33e1124711ad2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 00:51:08 +0800 Subject: [PATCH 093/465] knowledge(vllm-omni): sync f97af75d (PR #5792) Upstream-Commit: f97af75d1d7a0164674ad1f927db086d79f2b249 Upstream-PR: #5792 Knowledge-Delta: document batched MiniCPM-o Talker codec sampling --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minicpm-o-4-5/_index.md | 6 ++++- .../vllm-omni/models/minicpm-o-4-5/rules.md | 27 ++++++++++++++++++- 11 files changed, 43 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 423194b8c..31f71f80d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1a56953d7ac271d6a5afb9313e3c3f741f9e1a9a + previous_audited_sha: 73a312d5b6fe959318fd13dcd734f7b817b174fa audited_ref: main - audited_sha: 73a312d5b6fe959318fd13dcd734f7b817b174fa + audited_sha: f97af75d1d7a0164674ad1f927db086d79f2b249 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c8d9c03e9..44686604e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -73a312d5b6fe959318fd13dcd734f7b817b174fa`. The canonical machine baseline +f97af75d1d7a0164674ad1f927db086d79f2b249`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4c27b48a6..7c84183b5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 73a312d5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f97af75d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3241fc1af..8ab75dd96 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 73a312d5` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f97af75d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9da0c2e32..ca62b3bf2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 73a312d5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f97af75d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b74312661..91216200a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 73a312d5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f97af75d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index febb4f9af..abd2a6df4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 73a312d5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f97af75d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5f2c2cc22..5eb8f23bc 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 73a312d5` 验证存在 +- 源码校验：以上路径均已在 `main @ f97af75d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 13bb7e5e6..eb0c0d518 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 73a312d5`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f97af75d`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 1a56953d \ - --to 73a312d5 \ + --from 73a312d5 \ + --to f97af75d \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index e7d22f055..20ae90370 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5638", "PR #5869", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/] +sources: ["PR #3642", "PR #5382", "PR #5638", "PR #5792", "PR #5869", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py] confidence: high --- @@ -39,6 +39,10 @@ Thinker 的 Whisper/APM audio encoder 仍构造 dense `[B,1,T,T]` mask；chunk m broadcasted query/key index 代替逐 row Python fill，但不改变 chunk/left-context/lookahead 边界，也不消除 O(T²) storage。语义与证据门禁见 MCPMO-2b。 +Talker 只收集 sampling-eligible active rows 批量执行 codec projection 与 logit transforms， +同时保留逐请求 generator 的 row-wise multinomial；request alignment、terminal 与 workspace +边界见 MCPMO-3c。 + 描述直达源码与模型专有门禁见 [rules](rules.md#direct-代码快速入口)；新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 061e96037..b5bfc89ca 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5638", "PR #5869", "PR #6154", "PR #6170", "PR #6318", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5638", "PR #5792", "PR #5869", "PR #6154", "PR #6170", "PR #6318", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] confidence: high --- @@ -24,6 +24,7 @@ confidence: high | Code2Wav TensorRT、DiT/Campplus、engine cache/profile | MCPMO-1c | `minicpmo_4_5_code2wav.py::MiniCPMO45Code2Wav` → `batched_token2wav.py::BatchedToken2Wav._estimator_step`；共享实现 `step_audio2/step_audio2_dit_trt.py` | | HiFT CUDA Graph、chunk bucket、lazy capture、iSTFT | MCPMO-1d | `cuda_graph_wrapper.py::HiFTGraphWrapper` → `BatchedToken2Wav._hift_inference` → shared `HiFTGenerator` | | batch、`runtime_info`、stage handoff | MCPMO-3a/3b | `stage_input_processors/minicpmo_4_5_omni.py` → `minicpmo_4_5_omni.py::MiniCPMO45OmniForConditionalGeneration` → TTS/code2wav | +| Talker codec sampling、repetition penalty、request RNG/compaction | MCPMO-3c | `minicpmo_4_5_omni_tts.py::{make_omni_output,_sample_audio_codes,_apply_batched_repetition_penalty}` → `test_talker_batching.py` | | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | | instructions/persona/voice/mode update、prefill slots、context lock | MCPMO-4b | `experimental/fullduplex/minicpmo45/session.py` → runtime adapter/session runner | @@ -153,6 +154,30 @@ confidence: high - 禁止：直接 TTS class 不读 runtime bridge；返回 tuple 让 runner 猜它是 hidden state。 - 验收：真实 wrapper handoff 测试断言字段名、逐请求 shape、最终公开 payload。 ^[PR #3642] +## MCPMO-3c — Talker 批量 logit 处理必须保留逐请求采样与输出索引 + +- 触发：修改 Talker `make_omni_output`、codec projection、repetition penalty、top-k/top-p、 + multinomial、terminal 或 compaction。 +- 强制：只把有效 span、未 finished 且 sampling-eligible 的 row 收集为 pending samples；一次批量 + 执行 head projection、frequency-aware repetition penalty、top-k/top-p 与 softmax。每个 pending + item 必须携带原 `output_index`、request ID、history、step 与 state，采样结果再按该原索引写回 + codec delta、terminal 和 stop row。无效 info/span 与 incomplete prefill 保持 continue、空 delta， + 且不推进 request RNG/state；此前或本次 terminal 的请求才 stop。 +- RNG：`torch.multinomial` 每次只接收一个 generator，因此必须保留逐 row 调用各自 request-local + generator；禁止为了完全向量化而改成共享 generator。batch reorder、其他请求结束和 compaction + 均不得改变单请求序列。所有 row 采完后再把 concatenated tensor 一次搬到 CPU `.tolist()`， + 避免逐请求 `.item()` 同步。 +- workspace：repetition penalty 可保留 full-batch logits/result，但 encoded history 与 + `bincount` 必须最多按 16 rows 分块；token 编码为 `token + local_row*vocab`，count minlength + 只到 `chunk_rows*vocab`，使 int64 count 临时空间不随总并发一次放大。penalty=1 与空 batch + 保持 fast path，logits 必须为 2D且 histories/request IDs/steps 与 batch row 数一致。 +- 验收：逐 row oracle 对比正负 logits 与空 history，跨 16/16/1 chunk 固定 bincount workspace； + reorder、compaction 与 standalone 对照 request RNG；mixed eligible/ineligible/finished rows 断言 + 原索引 delta、stop、EOS/min/max token、duplex metadata 和只调用一次 batched sampler。CPU 单测 + 证明这些离散合同，但没有硬件上的 batched GEMM/logit 与旧 serial GEMV 数值/音频 parity；PR + 也没有 NPU concurrency 1/4 的 latency、吞吐或内存数据，因此不得宣称性能或质量不变/提升。 + ^[PR #5792] + ## MCPMO-4a — native duplex 保留可恢复 Stage0，VAD 只作显式策略 - 触发：MiniCPM-o native duplex 的连续音频、LISTEN/SPEAK handoff 或 server VAD interruption。 From 3863c0749a2a45ea3bd7b6dc37b95eb63adf0d01 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 00:59:07 +0800 Subject: [PATCH 094/465] knowledge(vllm-omni): sync 6d4e4ffc (PR #5850) Upstream-Commit: 6d4e4ffc7cf68f68b6d30076e04113737e35f567 Upstream-PR: #5850 Knowledge-Delta: qualify MiniMax H3 RTX 4090 deployment evidence --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/minimax-h3/_index.md | 2 +- .../models/minimax-h3/rules-deployment.md | 33 ++++++++++++++----- 11 files changed, 37 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 31f71f80d..b7549b507 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 73a312d5b6fe959318fd13dcd734f7b817b174fa + previous_audited_sha: f97af75d1d7a0164674ad1f927db086d79f2b249 audited_ref: main - audited_sha: f97af75d1d7a0164674ad1f927db086d79f2b249 + audited_sha: 6d4e4ffc7cf68f68b6d30076e04113737e35f567 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 44686604e..21fe88e21 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f97af75d1d7a0164674ad1f927db086d79f2b249`. The canonical machine baseline +6d4e4ffc7cf68f68b6d30076e04113737e35f567`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7c84183b5..5b40ecf83 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f97af75d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6d4e4ffc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8ab75dd96..157d71716 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f97af75d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6d4e4ffc` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ca62b3bf2..236676f73 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f97af75d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6d4e4ffc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 91216200a..ada32cf3d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f97af75d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6d4e4ffc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index abd2a6df4..0321578e1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f97af75d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6d4e4ffc` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5eb8f23bc..9265a43d1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f97af75d` 验证存在 +- 源码校验：以上路径均已在 `main @ 6d4e4ffc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index eb0c0d518..48438d571 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f97af75d`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6d4e4ffc`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 73a312d5 \ - --to f97af75d \ + --from f97af75d \ + --to 6d4e4ffc \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index f4c82c343..75ed4841d 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 24f77511b..17d347518 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5863", "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5863", "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -37,15 +37,30 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule - 触发：引用 H3 2×RTX 5090/4090 可运行性、延迟、峰值或 resident-layer 默认值。 - 强制：5090 证据只绑定 vLLM-Omni `ae6577ea` 的一次 2×32 GiB、T2VA、1344×768、 124 frames、50 steps run：8m38s，约 22.6 GiB/GPU 是 sampled `nvidia-smi`，不是 allocator - high-water。4090 的 2×24 GiB/1024×576/12 resident layers 只来自 2×B300 的 5-step - capacity proxy；它不是 RTX 4090 latency 或 full-run validation。 -- 禁止：把单次未 warmed 的 5090 run 写成 benchmark，把 B300 proxy 写成 target hardware， - 或从 HBM fit 推断 host fit。每个 FL2VA/Ref2VA partition 约 135 GiB，no-AllGather worker 的 - pinned CPU master 不因 resident layers 增加而消失；recipe 要求至少 200 GiB available RAM。 -- 验收：目标硬件用单一 partition、exact TP2/no-AllGather/VAE PP2/cuDNN/eager topology 复测， - 分别记录 allocator peak、重复 latency 和 joint video/audio quality。目标 pin 引用的 + high-water。早期 4090 容量依据只是 2×B300 的 5-step proxy；PR #5850 后新增的目标硬件 + 证据绑定 vLLM-Omni `81b48e83`（不是 merge target）、vLLM 0.26.0、PyTorch 2.11+cu130、 + driver 580.126.09、BF16/CUDNN/eager、TP2、no-AllGather DLO、12 resident layers、单 partition、 + 1024×576、124 frames@24 FPS、60 steps、seed 1101、video/audio shift 12/3。 +- 强制：2×4090 使用 USP1、text-encoder TP2、VAE PP2；T2VA 两次约 435/429 s、rank-0 allocator + reserved high-water 15.3 GiB，Ref2VA 两次约 892/892 s、14.6 GiB，两者均报告完整 ffmpeg decode。 + 4×4090 使用 USP2、text-encoder TP4、VAE PP4；T2VA 两次约 274/270 s、rank-0 15.2 GiB并完整 + decode；Ref2VA 只有一次 HTTP 200，约 545 s、rank-0 16.1 GiB，未声明完整 decode。后续 Ref2VA + 虽完成 diffusion，却在 D2H 阶段触发 engine hardcoded 30 s async-output wait，因此 4 卡 Ref2VA + repeatability 不成立。header peak 只表示 rank 0 `torch.cuda.max_memory_reserved`，不是所有 rank + 最大值；USP 不进一步 shard DiT weight，只能按该 workload 描述 per-GPU observation。 +- 禁止：把单次未 warmed 的 5090 run 写成 benchmark，把 B300 proxy 写成 target hardware，或从 + HBM fit 推断 host fit。每个 FL2VA/Ref2VA partition 约 135 GiB、至少 200 GiB available RAM与 + 推荐 384 GiB 只是 recipe 声明，未给 checkpoint revision/checksum 或 host high-water； + no-AllGather worker 的 pinned CPU master 不因 resident layers 增加而消失。两次输出大小相差不超过 + 115 bytes 不证明数值、感知或 A/V 质量一致；这些是 single-request latency，不是 concurrent + throughput。单卡 4090 未测；5090 的 26.5 GiB 单卡 profile 超过 24 GiB，而降低 resident layers + 的替代方案也未测。 +- 验收：新硬件/拓扑用单一 partition 与 exact config，记录 immutable model/environment、all-rank + allocator/整机峰值、warmup/repeats、stage/E2E latency 和 joint video/audio quality。recipe 的 + “after #5720 lands” 已过期，因为 modular H3 在该 target 已存在；partition-path 命令本身仍保持 + 单 partition，无需据此推断 combined route。目标 pin 引用的 `examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh` 实际不存在，因此不能把其 - four-task/MP4 validation 描述成可执行入口；补文件或改文档后再验收。^[PR #5764] + four-task/MP4 validation 描述成可执行入口；补文件或改文档后再验收。^[PR #5764] ^[PR #5850] ## MMH3-3c — H100 DFX fixture 只证明 exact nightly workload 与 payload path From bb05a5a73d4d0fdfe2850246986bdc89ff587a92 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 01:05:45 +0800 Subject: [PATCH 095/465] knowledge(vllm-omni): sync c4a14091 (PR #6029) Upstream-Commit: c4a14091c9b8fd61f4da46022f683adc1dc91550 Upstream-PR: #6029 Knowledge-Delta: repository skill contributor workflow and trust boundary --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/docs/_index.md | 3 +- .../repos/vllm-omni/docs/repository-skills.md | 53 +++++++++++++++++++ knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 11 files changed, 67 insertions(+), 13 deletions(-) create mode 100644 knowledge/repos/vllm-omni/docs/repository-skills.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b7549b507..216928a78 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f97af75d1d7a0164674ad1f927db086d79f2b249 + previous_audited_sha: 6d4e4ffc7cf68f68b6d30076e04113737e35f567 audited_ref: main - audited_sha: 6d4e4ffc7cf68f68b6d30076e04113737e35f567 + audited_sha: c4a14091c9b8fd61f4da46022f683adc1dc91550 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 21fe88e21..6c867381d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6d4e4ffc7cf68f68b6d30076e04113737e35f567`. The canonical machine baseline +c4a14091c9b8fd61f4da46022f683adc1dc91550`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5b40ecf83..9989c90a1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6d4e4ffc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c4a14091` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 157d71716..80ac64043 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6d4e4ffc` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c4a14091` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 236676f73..1459a00ba 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6d4e4ffc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c4a14091` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ada32cf3d..4bd621d4d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6d4e4ffc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c4a14091` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0321578e1..117d7a638 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6d4e4ffc` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c4a14091` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9265a43d1..606e9dd2b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6d4e4ffc` 验证存在 +- 源码校验：以上路径均已在 `main @ c4a14091` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/_index.md b/knowledge/repos/vllm-omni/docs/_index.md index a500bd9e4..8d7708f2b 100644 --- a/knowledge/repos/vllm-omni/docs/_index.md +++ b/knowledge/repos/vllm-omni/docs/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: index tags: [vllm-omni, docs] -sources: ["PR #5715", README.md, docs/configuration/README.md, docs/getting_started/installation/README.md, docs/getting_started/installation/gpu/cuda.inc.md, docs/getting_started/installation/npu/npu.inc.md] +sources: ["PR #5715", "PR #6029", .claude/skills/readme.md, README.md, docs/configuration/README.md, docs/contributing/README.md, docs/getting_started/installation/README.md, docs/getting_started/installation/gpu/cuda.inc.md, docs/getting_started/installation/npu/npu.inc.md] --- # vLLM-Omni 文档 @@ -24,6 +24,7 @@ sources: ["PR #5715", README.md, docs/configuration/README.md, docs/getting_star | 判断 RFC 是否仍在进行 | [RFC status](rfcs/_index.md) | | 找上游官方设计文档与其知识树 owner | [design-doc map](design-doc-map.md) | | generated example URL/nav、supported-model recipe 与硬件证据 | [generation and support rules](generation-and-support-rules.md) | +| contributor agent workflow、repository skill catalog 与信任边界 | [repository skill rules](repository-skills.md) | ## 发布与安装边界 diff --git a/knowledge/repos/vllm-omni/docs/repository-skills.md b/knowledge/repos/vllm-omni/docs/repository-skills.md new file mode 100644 index 000000000..426a40111 --- /dev/null +++ b/knowledge/repos/vllm-omni/docs/repository-skills.md @@ -0,0 +1,53 @@ +--- +title: "Repository skill contributor rules" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, docs] +sources: ["PR #6029", .claude/skills/readme.md, .claude/skills/add-diffusion-model/SKILL.md, .claude/skills/add-tts-model/SKILL.md, .claude/skills/diffusion-perf-opt/SKILL.md, .claude/skills/precheck-pr/SKILL.md, .claude/skills/quantization/SKILL.md, .claude/skills/review-pr/SKILL.md, .claude/skills/vllm-omni-npu-upgrade/SKILL.md, .claude/skills/vllm-omni-test/SKILL.md, docs/contributing/README.md] +confidence: high +--- + +# Repository skill contributor rules + +只有 `DOCSKILL-数字字母` 是可审计规则 ID。这里拥有 contributor-facing catalog、组合流程和 +信任边界；具体 review 行为仍由 [review owner](../review/_index.md) 管理，各 domain 的源码合同 +仍回对应 model/component owner。 + +## DOCSKILL-1a — catalog 必须从目标 revision 的真实 skill 目录生成 + +- 触发：增删/重命名 `.claude/skills/*`、修改 frontmatter `name`，或更新 contributor task-to-skill + table 与 `.claude/skills/readme.md`。 +- 强制：以目标 revision 的目录与每份 `SKILL.md` frontmatter 为准，分别校验链接路径和 invocation + name。`main @ c4a14091` 恰有 8 个目录且都有 `SKILL.md`：`add-diffusion-model`、 + `add-tts-model`、`diffusion-perf-opt`、`precheck-pr`、`quantization`、`review-pr`、 + `vllm-omni-npu-upgrade`、`vllm-omni-test`。NPU 的目录/链接名是 + `vllm-omni-npu-upgrade`，frontmatter 与展示的调用名是 + `vllm-omni-npu-model-runner-upgrade`，两者不能互换。旧 catalog 的 `add-omni-model` 与 + `generate-release-note` 没有目标目录，不能继续列为可用 skill。 +- 禁止：只检查显示名称而不解析链接；从文档名称推断存在安装包、hook、agent registry 或 runtime + registration。contributor guide 的 GitHub 链接指向可变 `main`；发布审计必须读取 pinned target + SHA，不能把未来 `main` 内容倒灌到当前合同。 +- 验收：对 catalog/task table 的每行做 link-exists + frontmatter-name 对照，并反向检查所有顶层 + skill 目录都被登记。目标提交只改两份 Markdown，没有修改 skill body、reference、script、test + 或 agent runtime；报告的 strict MkDocs/pre-commit 证明文档可构建，不证明 automatic discovery、 + invocation 或行为正确，且没有 catalog link/frontmatter 的自动回归。 + +## DOCSKILL-1b — skill 是可审查指令，不是授权、执行证明或安全边界 + +- 触发：贡献者要求 coding agent 使用 repository skill，组合 domain/test/precheck workflow，或引用 + skill 输出作为合并证据。 +- 强制：issue、accepted RFC 或明确 maintainer decision 定义任务权限与预期行为；先选最窄 domain + skill，生产行为或 tests/CI 改动再加 `vllm-omni-test`，提交前用 `precheck-pr`。要求 agent 给出 + 精确可运行命令、结果和所有因硬件、权重、credential、dependency 缺失而未运行的检查；人工再 + 检查 diff scope、无关改动、测试、文档以及性能/精度证据。maintainer review 仍不可省略。 +- 信任边界：automatic discovery 取决于具体 agent；手工指向 `SKILL.md` 只提供仓库维护者写入的 + 指令文本，不保证 agent 已安装/识别、会遵守、能调用所需工具或实际执行命令。读取 skill 不会 + 自动扩展用户授权，也不形成 filesystem/network/credential sandbox；执行前仍须审查目标 revision + 的 skill、references 与 scripts，按 agent/platform 的权限机制处理副作用和秘密。 +- 禁止：把“ask it to use”描述成强制 enforcement；把未运行检查写成 pass；用 `precheck-pr` 代替 + CI、maintainer review 或 accepted design；因为多个 skill 可组合就让它们越过 issue scope，或 + 假设不同 coding agent 对 `.claude/skills` 有相同 discovery/invocation 语义。 +- 验收：交付记录 task authority、选用的 skill 与理由、实际命令/结果、未运行项、diff review 和 + reviewer owner；若要声称跨 agent 可发现/执行，必须增加至少一个受支持 agent 的 pinned-version + behavioral eval。PR #6029 没有此类 agent eval，因此只建立 contributor guidance。^[PR #6029] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 48438d571..70209d7de 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6d4e4ffc`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c4a14091`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from f97af75d \ - --to 6d4e4ffc \ + --from 6d4e4ffc \ + --to c4a14091 \ --repo \ --mode report-only ``` From a5d6ea1776dcf44a4f73ae13f8aa24cc6159ac14 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 01:17:00 +0800 Subject: [PATCH 096/465] knowledge(vllm-omni): sync 322600bf (PR #5524) Upstream-Commit: 322600bfb2008d28fb94cfef73652dca63dac8dd Upstream-PR: #5524 Knowledge-Delta: MiniCPM-o accuracy, realtime Seed-TTS, CI, and config boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 6 ++++- .../components/configuration/_index.md | 2 +- .../configuration/omni-init-args.md | 11 +++++--- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minicpm-o-4-5/_index.md | 5 +++- .../vllm-omni/models/minicpm-o-4-5/rules.md | 27 ++++++++++++++++++- 13 files changed, 55 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 216928a78..7a00fbb56 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6d4e4ffc7cf68f68b6d30076e04113737e35f567 + previous_audited_sha: c4a14091c9b8fd61f4da46022f683adc1dc91550 audited_ref: main - audited_sha: c4a14091c9b8fd61f4da46022f683adc1dc91550 + audited_sha: 322600bfb2008d28fb94cfef73652dca63dac8dd # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6c867381d..f6ed3bba1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c4a14091c9b8fd61f4da46022f683adc1dc91550`. The canonical machine baseline +322600bfb2008d28fb94cfef73652dca63dac8dd`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 7b632e6ba..431b83d7a 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .pre-commit-config.yaml, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -123,4 +123,8 @@ confidence: high Hunyuan 配置只保留 H200 baseline，却仍标记 H100/A3 执行，当前因无阈值断言而被掩盖。 runtime alias 匹配还应保持最长 token 优先；目标 pin 中 `B200` 早于 `GB200`，会把 GB200 名称 归一化成 B200，修复前不可把 filename label 当精确设备证据。 + MiniCPM-o 4.5 的 nightly perf 是这个边界的具体实例：最终 simplex/duplex 配置都是单卡 H100， + runner 命令没有阈值开关；simplex 只 gate 完成请求数，duplex 另要求每 session 恰好四个音频 + response。配置里的 H100 baseline 因此只是结果 artifact，既不能证明回归阈值，也不能外推到 + A3；PR 文本中的“双卡”描述也不能覆盖最终 lane/YAML 的单卡事实。^[PR #5524] ^[PR #5402] ^[PR #5845] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9989c90a1..343ea2c0e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c4a14091` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 322600bf` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/omni-init-args.md b/knowledge/repos/vllm-omni/components/configuration/omni-init-args.md index 44946bde6..68ffd2435 100644 --- a/knowledge/repos/vllm-omni/components/configuration/omni-init-args.md +++ b/knowledge/repos/vllm-omni/components/configuration/omni-init-args.md @@ -1,10 +1,10 @@ --- title: "Legacy stage YAML 与顶层 EngineArgs" created: 2026-07-16 -updated: 2026-07-29 +updated: 2026-09-02 type: guide tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45"] +sources: ["PR #5524", "claude-workflow-starter-private@296ea45", tests/engine/test_arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/arg_utils.py, vllm_omni/entrypoints/utils.py] --- # Legacy stage YAML 与顶层 EngineArgs @@ -17,7 +17,7 @@ sources: ["claude-workflow-starter-private@296ea45"] ## 当前合同 -`stage_configs_path` 存在时，`AsyncOmniEngine._resolve_stage_configs` 会在加载 YAML 前调用 `_strip_single_engine_args`： +legacy `stage_configs_path` 存在时，`AsyncOmniEngine._resolve_stage_configs` 会在加载 YAML 前调用 `_strip_single_engine_args`： 1. `vllm` 顶层 `EngineArgs` 的普通默认值和非默认值不会直接泄漏进每个 stage。 2. 除 `_PARENT_ARGS_NO_WARN` 明确豁免的字段外，被忽略的显式非默认值会记录 warning，不能静默覆盖 YAML。 @@ -26,6 +26,11 @@ sources: ["claude-workflow-starter-private@296ea45"] 当前过滤 primitive 在 `vllm_omni/config/stage_config.py::strip_parent_engine_args`，入口和 allowlist 在 `vllm_omni/engine/async_omni_engine.py::_strip_single_engine_args` 及 `_PARENT_ARGS_*`。CLI 字段 owner 则由 `vllm_omni/engine/arg_utils.py::OrchestratorArgs`、`SHARED_FIELDS` 和 `internal_blacklist_keys` 声明。 +new-format deploy YAML 是另一条合同：识别为 new format 后必须保留 parent CLI 字段进入配置 +合并，尤其是 `interleave_mm_strings` 与 `media_io_kwargs`；不能复用 legacy `stage_args` 的 stripping +路径，否则 CLI 的 interleave/media override 会在 stage 构造前静默丢失。只有 legacy schema 才 +执行上述 `_strip_single_engine_args`。^[PR #5524] + ## 修改时怎样验证 - 不要再依赖或推荐已不存在的 `nullify_stage_engine_defaults`。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 80ac64043..9a469b651 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c4a14091` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 322600bf` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1459a00ba..a20f4a62a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c4a14091` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 322600bf` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4bd621d4d..121483ffe 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c4a14091` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 322600bf` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 117d7a638..1e118a515 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c4a14091` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 322600bf` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 606e9dd2b..df4bcdb3d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c4a14091` 验证存在 +- 源码校验：以上路径均已在 `main @ 322600bf` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 70209d7de..956524eb5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c4a14091`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 322600bf`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 6d4e4ffc \ - --to c4a14091 \ + --from c4a14091 \ + --to 322600bf \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 20ae90370..a89da2901 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5638", "PR #5792", "PR #5869", "PR #6154", "PR #6170", "PR #6318", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py] +sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py] confidence: high --- @@ -43,6 +43,9 @@ Talker 只收集 sampling-eligible active rows 批量执行 codec projection 与 同时保留逐请求 generator 的 row-wise multinomial；request alignment、terminal 与 workspace 边界见 MCPMO-3c。 +Daily-Omni、Seed-TTS simplex/realtime duplex 已进入 accuracy/perf harness；阈值、逐轮 metrics、 +单卡 H100 perf 与“baseline 不是 regression gate”的证据边界见 MCPMO-5a。 + 描述直达源码与模型专有门禁见 [rules](rules.md#direct-代码快速入口)；新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index b5bfc89ca..731a0f4c9 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5638", "PR #5792", "PR #5869", "PR #6154", "PR #6170", "PR #6318", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] confidence: high --- @@ -27,6 +27,7 @@ confidence: high | Talker codec sampling、repetition penalty、request RNG/compaction | MCPMO-3c | `minicpmo_4_5_omni_tts.py::{make_omni_output,_sample_audio_codes,_apply_batched_repetition_penalty}` → `test_talker_batching.py` | | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | | instructions/persona/voice/mode update、prefill slots、context lock | MCPMO-4b | `experimental/fullduplex/minicpmo45/session.py` → runtime adapter/session runner | +| Daily-Omni/Seed-TTS accuracy、simplex/duplex perf、TTFT/TTFP/RTF | MCPMO-5a | `tests/e2e/accuracy/minicpmo_4_5/`、`tests/dfx/perf/tests/test_minicpmo_4_5*.json` → `run_benchmark.py` | ## MCPMO-1a — trust_remote_code 服从用户选择 @@ -199,5 +200,29 @@ confidence: high - 验收：成功更新同时改变派生状态；资源不足时无部分 mutation；buffer/defer/reject 均不锁， append 成功才锁，mode flip 明确拒绝。 ^[PR #6318] +## MCPMO-5a — accuracy 与 duplex perf 证据必须绑定协议、硬件和实际 gate + +- 触发：修改 MiniCPM-o 4.5 Daily-Omni、Seed-TTS、realtime duplex 数据集/客户端、benchmark + 配置或 CI lane。Daily-Omni 合同是 `minicpm-interleave`、temperature 0、文本输出、512 tokens， + 使用 TTS template 与 1 FPS/最多 128 帧 media 配置，准确率下限 0.78；simplex Seed-TTS 同时请求 + text+audio，WER 上限 0.05。duplex Seed-TTS 走 `openai-realtime-tts`，每 session 四轮、50 prompts， + 并要求每轮恰好一个音频 response。 +- 强制：多轮数据只把连续有效 row 按 `turns_per_session` 分组，沿用首行 reference audio/text； + 多轮必须使用 realtime backend，在同一 session 依次发送 item/response create 并做 playback ACK。 + 客户端按轮从共同 measurement origin 单调记录首 text TTFT、首 audio TTFP、逐轮请求起点到末音频的 RTF； + PCM 与这些 metrics 必须展开回逐轮 WER 样本。`native_duplex=False` 必须显式走 chat fallback，且在 + 构造 `ChatCompletionRequest` 前移除 `minicpmo45_native_duplex`。 +- 证据边界：accuracy case 标为 H100 或 A3、每 case 一卡，但 nightly 可为第二张卡上的 WER + evaluator 分配两卡资源；资源 allocation 不等于模型 stage 用卡数。最终 simplex/duplex perf 是 + 单卡 H100，baseline 只有 H100 bucket，runner 只检查完成数，duplex 再检查四轮 audio count；没有 + 性能阈值 consumer，故 baseline、TTFT/TTFP/RTF 只构成测量 artifact，不能证明回归或 A3 性能。 +- 禁止：把结果 artifact 中存在 baseline 描述成 active performance gate；把 PR 的“双卡”文字覆盖到 + 最终单卡 lane/YAML；或从 H100 bucket 外推 A3 的性能与阈值。 +- 验收：固定上述 collection、阈值、逐轮 cardinality 与 fallback；分别故障注入缺轮、重复音频、 + 非单调时间和错误 backend。PR 报告的 Daily-Omni 78.53%、Seed-TTS WER 0.0332/0.0255 只能绑定其 + 日志环境：数据/模型 revision 未固定，Daily 首次受损坏 decord 失败后重跑，kernel warmup 另有 + 未提交 patch，证据脚本还吞掉 step failure，且把版本号写作 commit。因而不能从这些数字声称 + merge target 的可复现质量/性能通过。^[PR #5524] + 共享 bridge/batch 规则见 [Model Executor rules](../../components/model-executor/rules.md)； 公开入口完整性见 [model adaptation guardrails](../../review/guides/model-adaptation-guardrails.md)。 From cff9021f8854892fdb36b2d2ab0d36245e4cbfe1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 01:27:03 +0800 Subject: [PATCH 097/465] knowledge(vllm-omni): sync af3faa6c (PR #5550) Upstream-Commit: af3faa6c1e502c92e79637eab15485b7cc5bf356 Upstream-PR: #5550 Knowledge-Delta: diffusion paged-KV W0 metadata transport and init cleanup --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/paged-kv-control-plane.md | 32 ++++++++++++++++--- .../diffusion/rules-output-lifecycle.md | 12 ++++++- .../vllm-omni/components/diffusion/rules.md | 10 +++--- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 12 files changed, 56 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7a00fbb56..29d5ac3da 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c4a14091c9b8fd61f4da46022f683adc1dc91550 + previous_audited_sha: 322600bfb2008d28fb94cfef73652dca63dac8dd audited_ref: main - audited_sha: 322600bfb2008d28fb94cfef73652dca63dac8dd + audited_sha: af3faa6c1e502c92e79637eab15485b7cc5bf356 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f6ed3bba1..ede4f72ff 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -322600bfb2008d28fb94cfef73652dca63dac8dd`. The canonical machine baseline +af3faa6c1e502c92e79637eab15485b7cc5bf356`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 343ea2c0e..5f54ed1a0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 322600bf` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ af3faa6c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9a469b651..aeb6ac564 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 322600bf` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ af3faa6c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md b/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md index 2b8e4e04c..754a63839 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md +++ b/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion, scheduler] -sources: ["PR #5541", vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/diffusion_kv/config.py, vllm_omni/diffusion/diffusion_kv/request.py, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py, vllm_omni/diffusion/request.py, vllm_omni/diffusion/sched/base_scheduler.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/utils.py, tests/config/test_omni_config.py, tests/diffusion/diffusion_kv/test_config.py, tests/diffusion/diffusion_kv/test_request.py, tests/diffusion/models/hunyuan_image3/test_diffusion_kv_request.py, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/test_diffusion_config_propagation.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_model_runner.py, tests/diffusion/test_diffusion_scheduler.py] +sources: ["PR #5541", "PR #5550", vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/diffusion_kv/config.py, vllm_omni/diffusion/diffusion_kv/metadata.py, vllm_omni/diffusion/diffusion_kv/request.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py, vllm_omni/diffusion/request.py, vllm_omni/diffusion/sched/base_scheduler.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/worker/utils.py, tests/config/test_omni_config.py, tests/diffusion/diffusion_kv/test_config.py, tests/diffusion/diffusion_kv/test_metadata.py, tests/diffusion/diffusion_kv/test_request.py, tests/diffusion/diffusion_kv/test_worker_contract.py, tests/diffusion/models/hunyuan_image3/test_diffusion_kv_request.py, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/test_diffusion_config_propagation.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_model_runner.py, tests/diffusion/test_diffusion_scheduler.py] --- # Diffusion paged KV control plane @@ -68,10 +68,34 @@ sources: ["PR #5541", vllm_omni/config/omni_config.py, vllm_omni/diffusion/data. native manager facade test 不得冒充 Scheduler allocation、BlockTable、slot mapping、paged attention 或跨请求正确性。^[PR #5541] +## DIFFKV-2c — Worker 只接收请求级 allocation snapshot，不接收 Scheduler 可变状态 + +- 触发：增加 `DiffusionKVMetadata`、`NewRequestData` 字段，或修改 request/batch/step 的 + Scheduler→Executor→Worker→ModelRunner RPC。snapshot 在 request scope 保存 generation、sequence + 的 prefix/target/full length、可引用的独立 context，以及每个 native cache group 的 block ID； + 多个 CFG sequence 可引用同一个 context，也可引用各自 context。 +- 强制：metadata 与已初始化 request 原子放在同一个 new-request envelope，且 envelope ID、实际转发 + 的 `req.request_id`、metadata ID 在任何 model work/RPC 前一致。request mode 只在 metadata 存在时 + 追加第四个 RPC 位置参数，保持 dense 历史三参数 shape；batch/step 则随完整 + `DiffusionSchedulerOutput` 传输。`prepared_layout` 仍是独立的 model-owned payload，不能从它派生 + block ID。Worker wrapper 也只在非空时向旧 runner 增加 keyword，避免破坏 dense/custom runner。 +- 强制：W0 尚无 allocator/installer，因此 `paged_scheduler` 接受 metadata 但不得强制要求； + `dense_legacy` 必须拒绝 metadata。带 metadata 的多请求 wave 不能走 DLO 的 list-request 快捷 RPC， + 因为该路径没有逐请求 snapshot 参数；应退回逐请求 dispatch。^[PR #5550] +- 禁止：把这些可变 dataclass 当成已验证的安全边界。当前只验证 mode 与顶层 request identity， + 不检查 generation/sequence/block ID 正数、`prefix+target<=seq_len`、精确 sequence/context 集合、 + cache-group 数/顺序、block-table 长度/范围或 null block。上述检查必须由未来 Scheduler producer + 对照 `SchedulerRequestState.diffusion_kv_requests`，再由 Worker installer 对照 rank-local native + `KVCacheConfig` 与 generation registry，而不是用 DTO 自证正确。 +- 验收：request/batch/step 覆盖 metadata 缺省、传播与三方 ID mismatch；dense metadata fail-fast， + DLO metadata wave 不进入 list-request 分支，legacy positional shape 与 `prepared_layout` identity + 保持。严格 layout/range/generation 测试只能在独立 truth owner 落地后升级为已实现合同。 + ## 证据边界 -- 目标实现只提供 logical request/control-plane contracts 与 native-manager conformance tests；没有 - cache manager、physical allocation、Scheduler→Worker KV metadata、BlockTable、slot mapping 或 - paged attention。 +- 目标实现只提供 logical request/control-plane contracts、request-scoped Scheduler→Worker metadata + DTO/RPC 与 native-manager conformance tests；没有 cache manager、physical allocation、Worker + installer/registry、BlockTable、slot mapping 或 paged attention。metadata 可选且未被消费，内部 + 数值字段也未做严格校验，因此不能把“到达 runner”解释为 allocation 可用。 - 目标 Hunyuan E2E 运行 `dense_legacy`，只证明既有 dense path；不能作为 Scheduler-managed paged data plane、attention 或 cross-request caching 正确性的证据。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index d7ce9ed2e..0e5bb2cad 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -51,6 +51,16 @@ confidence: high 且没有后续强制 thread/SHM/queue 回收证明。final 2×B300 rerun提供一次普通 TP2 与 DLO DP2 clean exit，但不替代 fault matrix。^[PR #5864] +## DIFF-1j — Engine 构造失败必须释放已启动的 Executor + +- 触发：`DiffusionEngine.__init__` 在 executor 已启动后初始化 Scheduler、runtime state、execute + function 或 logging。 +- 强制：任一步失败都 best-effort 关闭已赋值 Scheduler、shutdown executor，再原样重抛根因； + cleanup 异常只能记录。此时同步状态可能尚未建立，不能调用完整 `close()`。 +- 禁止：构造异常直接逃逸并遗留 worker/monitor/GPU allocation，或用 cleanup 异常遮蔽初始错误。 +- 验收：注入 Scheduler initialization failure，断言 close/shutdown 各一次且异常 identity 不变； + 另覆盖 cleanup 自身失败不遮蔽根因。^[PR #5550] + ## DIFF-2k — DLO AllGather DP wave 必须完整兼容或不可复用地 fail closed - 触发：DLO+AllGather、DP>1 允许 concurrent request，或修改 multi-rank `execute_model` RPC。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 89570d046..f74a41bf9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -20,13 +20,13 @@ confidence: high | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| -| CUDA Graph、compile、fused solver/norm/RoPE、FA determinism、eager parity、tensor dtype/device、async output/teardown | `execution-parity`：`DIFF-1a`–`1i` | `compile.py::regionally_compile` → shared layer/backend → model denoise/output pump/shutdown | +| CUDA Graph、compile、fused solver/norm/RoPE、FA determinism、eager parity、tensor dtype/device、async output/teardown | `execution-parity`：`DIFF-1a`–`1j` | `compile.py::regionally_compile` → shared layer/backend → model denoise/output pump/shutdown | | seed、request-local generator、guidance=0、并发 RNG、batched generators | `execution-parity`：`DIFF-1b` | `inputs/data.py::OmniDiffusionSamplingParams` → runner `_initialize_generator` → request-batch generator collate | | ModelOpt/checkpoint adapter、weight/scale remap、unknown tensor、resolution path | `checkpoint-distributed`：`DIFF-2a` | `diffusers_loader.py::{_get_checkpoint_adapter,load_weights}` → `modelopt.py::{_resolve_target_and_output_names,adapt}` | | host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | | HSDP/FSDP、`fully_shard`、DeviceMesh、packed/scalar parameter、FP8 | `checkpoint-distributed`：`DIFF-2b` | `distributed/hsdp.py::{apply_hsdp_to_model,shard_model}` → loader `_load_model_with_hsdp` → `hsdp_fp8.py` | | distributed layerwise offload、AllGather、异构 block、shared buffer | `checkpoint-distributed`：`DIFF-2e` | `offloader/distributed_layerwise_backend.py::{DistributedLayerwiseOffloadHook.initialize_hook,prefetch_layer,DistributedLayerwiseOffloadBackend._allocate_shared_buffers}` | -| DLO+AllGather、DP concurrent collective wave、per-worker result queue、shutdown | `checkpoint-distributed`/`execution-parity`：`DIFF-2k`, `DIFF-1d/1g/1h` | request compatibility key → multi-rank RPC → worker-owned queue/pump → bounded teardown | +| DLO+AllGather、DP wave、result queue、shutdown、constructor cleanup | `checkpoint-distributed`/`execution-parity`：`DIFF-2k`, `DIFF-1d/1g/1h/1j` | request compatibility key → multi-rank RPC → worker-owned queue/pump → bounded teardown | | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | | modular/multi-DiT、`_dit_modules`、request-scoped Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2j` | request batch key → model policy/cache runtime；pipeline component list → shared consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | @@ -36,7 +36,7 @@ confidence: high | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | -| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1i` | +| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1j` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2k` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j` | @@ -75,8 +75,8 @@ confidence: high - 禁止：依赖 process 默认 fp32/CPU 或隐式 cast/move；只测 shape 而不执行 BF16 consumer。 - 验收：BF16、错长、batch reorder 和 graph/eager 用例断言 consumer 收到相同 device/dtype、 alias 与结果。 ^[PR #5067] ^[PR #5068] ^[PR #5174] ^[PR #5981] -async output readiness、per-worker result channel 与 shutdown 合同见 -[DIFF-1d/1g/1h 专页](rules-output-lifecycle.md)。 +async output readiness、per-worker result channel、shutdown 与 constructor cleanup 合同见 +[DIFF-1d/1g/1h/1j 专页](rules-output-lifecycle.md)。 ### DIFF-1e — fused layer 必须保留精度、布局与平台分派合同 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a20f4a62a..916a4e678 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 322600bf` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ af3faa6c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 121483ffe..094845e66 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 322600bf` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ af3faa6c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1e118a515..bebb992a0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 322600bf` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ af3faa6c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index df4bcdb3d..6fae2a1a6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 322600bf` 验证存在 +- 源码校验：以上路径均已在 `main @ af3faa6c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 956524eb5..73721dd8c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 322600bf`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ af3faa6c`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from c4a14091 \ - --to 322600bf \ + --from 322600bf \ + --to af3faa6c \ --repo \ --mode report-only ``` From a181bdf43b9448116974ec6e2c7885561ba32f25 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 01:37:19 +0800 Subject: [PATCH 098/465] knowledge(vllm-omni): sync 06e733d3 (PR #4099) Upstream-Commit: 06e733d3cd4400bb0029f8418e2207f1441a39c0 Upstream-PR: #4099 Knowledge-Delta: LongCat-Video-Avatar-1.5 offline AI2V/AT2V and AVC contracts --- adapters/vllm_omni/release_baseline.yaml | 8 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 10 +-- .../models/longcat-video-avatar-1-5/_index.md | 37 +++++++++ .../models/longcat-video-avatar-1-5/rules.md | 82 +++++++++++++++++++ 11 files changed, 135 insertions(+), 16 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/longcat-video-avatar-1-5/_index.md create mode 100644 knowledge/repos/vllm-omni/models/longcat-video-avatar-1-5/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 29d5ac3da..bae3cee65 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 322600bfb2008d28fb94cfef73652dca63dac8dd + previous_audited_sha: af3faa6c1e502c92e79637eab15485b7cc5bf356 audited_ref: main - audited_sha: af3faa6c1e502c92e79637eab15485b7cc5bf356 + audited_sha: 06e733d3cd4400bb0029f8418e2207f1441a39c0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 79 sha256: 700ab6a4918de2d796604d1e6da5f619731a53bb09b8c56416f253e820ffdf3a diffusion: - count: 59 - sha256: b6ee711d67f4c5daca6b1c6b5d7091bd61e1f978cb6a6548b79ff5700cafa041 + count: 60 + sha256: ae57a7d6867352714c00b5b93a5f22874ebc2f370cd32067925bb14ec84214f8 pipelines: count: 52 sha256: 54a31adef93bc33e2009252699075930ee05ffb38605a52ba9370a21c542b320 diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ede4f72ff..516fa7d95 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -af3faa6c1e502c92e79637eab15485b7cc5bf356`. The canonical machine baseline +06e733d3cd4400bb0029f8418e2207f1441a39c0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5f54ed1a0..2579da9d6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ af3faa6c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 06e733d3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index aeb6ac564..549d9f440 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ af3faa6c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 06e733d3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 916a4e678..2203a9b06 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ af3faa6c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 06e733d3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 094845e66..5fac29b2c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ af3faa6c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 06e733d3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bebb992a0..6357b6ca6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ af3faa6c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 06e733d3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6fae2a1a6..308f8dfc4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ af3faa6c` 验证存在 +- 源码校验：以上路径均已在 `main @ 06e733d3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 73721dd8c..67a04d817 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ af3faa6c`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 06e733d3`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -36,7 +36,7 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 79 个架构名 / 28 个模型族目录 | -| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 58 条 pipeline / 38 个模型族目录 | +| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 60 条 pipeline / 39 个模型族目录 | | Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 52 个 key | | Deploy YAML | `vllm_omni/deploy/*.yaml` | 80 份 | @@ -61,7 +61,7 @@ voxcpm2、voxtral_tts audiox、bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamid_omni、 dreamzero、ernie_image、flux、flux2、flux2_klein、glm_image、gr00t、helios、 hidream_image、hunyuan_image3、hunyuan_video、internvla_a1、krea2、lance、 -lingbot_video、longcat_image、ltx2、magi_human、ming_flash_omni、nextstep_1_1、omnigen2、 +lingbot_video、longcat_image、longcat_video、ltx2、magi_human、ming_flash_omni、nextstep_1_1、omnigen2、 omnivoice、ovis_image、qwen_image、sd3、sdxl、sensenova_u1、soulx_singer、 stable_audio、wan2_2、z_image、minimax_h3 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 322600bf \ - --to af3faa6c \ + --from af3faa6c \ + --to 06e733d3 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/longcat-video-avatar-1-5/_index.md b/knowledge/repos/vllm-omni/models/longcat-video-avatar-1-5/_index.md new file mode 100644 index 000000000..3d0433085 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/longcat-video-avatar-1-5/_index.md @@ -0,0 +1,37 @@ +--- +title: "LongCat-Video-Avatar-1.5" +created: 2026-09-02 +updated: 2026-09-02 +type: index +tags: [vllm-omni, models, diffusion] +sources: ["PR #4099", docs/models/supported_models.md, examples/offline_inference/longcat_video/end2end.py, recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md, vllm_omni/diffusion/models/longcat_video/, vllm_omni/diffusion/registry.py] +confidence: high +--- + +# LongCat-Video-Avatar-1.5 + +## 名称与范围 + +- 正式 checkpoint：`meituan-longcat/LongCat-Video-Avatar-1.5`；diffusion registry key + `LongCatVideoAvatarPipeline` 指向 `diffusion/models/longcat_video/`，并注册模型专属 pre/post process。 +- 单 stage、offline、单设备 Avatar 1.5：支持单说话人 AT2V/AI2V、两说话人 AI2V，以及单/两说话人 + 多 segment AVC continuation。默认 INT8 DiT、distilled LoRA、480p、93 frames、25 FPS、8 steps。 +- 不包含基础 LongCat-Video T2V/I2V、旧 Avatar checkpoint、online serving、refinement/super-resolution/ + high-frequency 路径、BSA、SP/TP/CFG parallel 或 Cache-DiT/TeaCache/FBCache。 + +## 运行与资源边界 + +Avatar checkpoint 提供 DiT、scheduler、Whisper、vocal separator 与 distilled LoRA；tokenizer、UMT5 +text encoder 和 VAE 来自独立的基础 `meituan-longcat/LongCat-Video`。额外运行依赖由 +`longcat-video-avatar` extra 安装。官方 recipe 和 advanced E2E 都绑定单张 H100；recipe 的约 +41.0/56.8 GiB 峰值分别只绑定 CPU/GPU component build、INT8+distill、93-frame AI2V exact case， +是容量提示而非性能或质量 gate。 + +输入、加载、AVC cache 与证据边界见 [规则](rules.md)。共享 diffusion 不变量见 +[Diffusion owner](../../components/diffusion/_index.md)。 + +## 什么时候查这里 + +- 修改 LongCat Avatar registry、checkpoint/LoRA/INT8 loading、audio/image/mask、AVC continuation、 + model-local KV cache 或 offline example/tests。 +- 基础 LongCat-Image 仍查 [LongCat-Image owner](../longcat-image/_index.md)。 diff --git a/knowledge/repos/vllm-omni/models/longcat-video-avatar-1-5/rules.md b/knowledge/repos/vllm-omni/models/longcat-video-avatar-1-5/rules.md new file mode 100644 index 000000000..621348516 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/longcat-video-avatar-1-5/rules.md @@ -0,0 +1,82 @@ +--- +title: "LongCat-Video-Avatar-1.5 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #4099", docs/user_guide/diffusion_features.md, examples/offline_inference/longcat_video/end2end.py, pyproject.toml, recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md, tests/diffusion/models/longcat_video/test_longcat_video_avatar.py, tests/e2e/offline_inference/test_longcat_video_avatar.py, tests/model_tests/diffusion/test_alignment.py, vllm_omni/diffusion/models/longcat_video/longcat_video_avatar_transformer.py, vllm_omni/diffusion/models/longcat_video/pipeline_longcat_video_avatar.py, vllm_omni/diffusion/registry.py] +confidence: high +--- + +# LongCat-Video-Avatar-1.5 规则 + +只有 `LCVA-数字字母` 是可审计规则 ID。 + +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | 第一批源码 | +|---|---|---| +| registry、HF/local、INT8、LoRA、base model、dependency | LCVA-1a | registry；`prepare_longcat_video_avatar_model_for_omni`；pipeline `weights_sources/load_weights` | +| AT2V/AI2V、multi-speaker、audio_type、bbox/mask | LCVA-2a | pre-process → `_resolve_request_inputs` → audio/mask helpers → transformer attention | +| AVC、num_segments、conditioning frames、KV cache | LCVA-3a | `_prepare_audio_embeddings` → first segment → `_generate_avc` → transformer KV reuse | +| H100、memory、E2E、online/parallel/cache acceleration | LCVA-4a | advanced E2E → recipe → diffusion feature matrix | + +## LCVA-1a — Avatar 与基础模型资产分开解析，加载模式必须自洽 + +- 触发：registry、model preparation、HF allow-pattern、INT8/full-precision DiT、distilled LoRA 或 + `longcat-video-avatar` extra。只支持 `model_type=avatar-v1.5`；Avatar repo 提供 scheduler、所选 + `base_model_int8`/`base_model`、DMD LoRA、Whisper 与 vocal separator，基础 LongCat-Video repo + 提供 tokenizer、UMT5 text encoder 与 VAE。 +- 强制：registry 同时绑定 pipeline 与专属 pre/post process。DiT 用 `weights_sources` + + `AutoWeightsLoader`，INT8 loader 必须覆盖 parameter 和 registered buffer；选择 INT8 时不能同时下载 + full-precision weight tree。CPU component build 是降低初始化峰值的默认，最终组件仍迁移到设备。 +- 禁止：把基础 LongCat T2V/I2V 或旧 Avatar checkpoint 归入该 key；并发调用 `enable_loras()`。 + 当前 LoRA 通过 monkey-patch `module.forward` 且明确非线程安全，只适用于现有单线程 offline 初始化。 +- 验收：local 与 HF ID、INT8/full precision、parameter/buffer load、`.to(device)` 与 `.to(dtype)`、 + extra 缺失分别覆盖。`prepare_*` 会写 `model_index.json` 并复制 transformer config 到给定目录， + 因而路径必须可写且调用方应把它视为 mutation；HF `revision=None` 也不能当不可变资产证据。 + `use_distill=True` 时缺 DMD LoRA 当前会静默不安装，但 forward 仍强制 8 steps/双 guidance=1；这是 + 未封闭 gap，修复应 fail fast 或显式回退非 distill schedule，并加 local-checkpoint 缺 LoRA 测试。 + +## LCVA-2a — stage、speaker audio 与空间 mask 必须作为同一个输入合同 + +- 触发：audio/image、`stage`、official JSON、`audio_type`、bbox 或 speaker mask。无显式 stage 时有图 + 推断 AI2V、无图推断 AT2V；所有请求必须有 audio，AI2V 必须有 image。多个 audio track 只能走 + AI2V，因为 mask 定义在 reference image 上。 +- 强制：speaker key 用末尾数字排序；`para` 把各轨补零到最长/生成时长并行播放，`add` 把各轨放入 + 依次相接且其余位置为零的 timeline。显式 bbox 顺序是 `[y_min,x_min,y_max,x_max]` 并裁到图像； + person1/person2 都缺时默认左右分区，只缺一个则拒绝，背景 mask 是两者补集。 +- 禁止：把任意说话人数描述成已验证；目标 transformer/test 只闭合两个 speaker，额外 `others` + 是空间 mask，不是额外 audio-speaker 映射。也不能在多说话人 AT2V 中隐式忽略 mask。 +- 验收：单 speaker AT2V/AI2V、两 speaker AI2V 的 para/add、显式/默认 bbox、单边 bbox、缺 audio/image、 + 非法 stage/resolution 都在昂贵 model work 前验证；输出是 RGB PIL frame list，不把 example 最后的 + ffmpeg 音频 mux 当 pipeline output 合同。 + +## LCVA-3a — AVC cache 只复用 continuation conditioning，不是 diffusion-step acceleration + +- 触发：`num_segments>1`/`auto`、`num_cond_frames`、reference latent、`use_kv_cache` 或 + `offload_kv_cache`。frame 数先归一为 `4k+1`；auto 以首段 `num_frames/fps`、后续 stride + `(num_frames-num_cond_frames)/fps` 覆盖完整 audio，且必须满足 `num_cond_frames Date: Thu, 3 Sep 2026 01:44:01 +0800 Subject: [PATCH 099/465] knowledge(vllm-omni): sync 4ab1b4ff (PR #4645) Upstream-Commit: 4ab1b4ff9cfc2ca233cccb5a4ba1f845288278fc Upstream-PR: #4645 Knowledge-Delta: SP padding mask correctness and dense-path performance policy --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../components/diffusion/rules-sp-padding.md | 51 +++++++++++++++++++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../repos/vllm-omni/models/flux2/rules.md | 5 +- .../vllm-omni/models/hunyuan-video/_index.md | 7 ++- 12 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bae3cee65..84ffdf011 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: af3faa6c1e502c92e79637eab15485b7cc5bf356 + previous_audited_sha: 06e733d3cd4400bb0029f8418e2207f1441a39c0 audited_ref: main - audited_sha: 06e733d3cd4400bb0029f8418e2207f1441a39c0 + audited_sha: 4ab1b4ff9cfc2ca233cccb5a4ba1f845288278fc # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 516fa7d95..5e69b5efe 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -06e733d3cd4400bb0029f8418e2207f1441a39c0`. The canonical machine baseline +4ab1b4ff9cfc2ca233cccb5a4ba1f845288278fc`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2579da9d6..e17972a69 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 06e733d3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4ab1b4ff` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 549d9f440..39cae1dff 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 06e733d3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4ab1b4ff` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -32,6 +32,7 @@ sources: [] | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | +| SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | | video/audio mux、DLO DP wave、result queue、async pump、SHM ownership 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md b/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md new file mode 100644 index 000000000..182fe653b --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md @@ -0,0 +1,51 @@ +--- +title: "Diffusion SP padding 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #4645", docs/user_guide/diffusion/parallelism/sequence_parallel.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/sp_plan.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/hooks/sequence_parallel.py, vllm_omni/diffusion/models/flux2/flux2_transformer.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py] +confidence: high +--- + +# Diffusion SP padding 规则 + +只有 `SPPAD-数字字母` 是可审计规则 ID。 + +## SPPAD-1a — padding mask 是显式正确性/性能策略，不是无损优化 + +- 触发：修改 `DiffusionParallelConfig.mask_sp_padding`、SP `auto_pad`、forward context 的 + `sp_original_seq_len/sp_padding_size`，或模型传给 attention 的 hidden-state mask。 +- 强制：只有 sequence-parallel size > 1、确实发生 auto-padding 且 `mask_sp_padding=True` 时，才按 + global 原长度构造 bool mask，将尾部 padding 标 false 并传给模型 attention。默认 false 时不传 + hidden mask，让 attention 留在 dense fast path；必须 warning once，明确 padding 数、原长度、SP + degree、潜在数值差异与 strict opt-in 方法。无 padding 或 SP=1 不改变 mask 选择。 +- 禁止：把未 mask 的零 token 当数学等价；它们仍可参与 softmax，影响有效 token。也不能把 + `mask_sp_padding=False` 描述为没有 padding：模型 `_sp_plan` 的 `auto_pad=True` 已在 attention 前补齐 + sequence。文档列出的 consumer 是 Wan2.2、Wan2.2 VACE、Qwen-Image、Flux2 与 HunyuanVideo1.5， + 不能由 config 字段存在外推到其他 pipeline。 +- 验收：每个 consumer 至少覆盖 SP=1、可整除 SP、不可整除且 mask true/false；断言 false 不传 mask + 并走 dense backend，true 只遮尾部 pad 并走对应 masked/varlen path，同时比较有效 token/output 的 + 有界数值差异。PR #4645 只修改 Flux2/HunyuanVideo1.5 源码与文档，没有新增自动测试。 + +## SPPAD-1b — advanced UAA 与模型前置 auto-padding 属于不同层 + +- 触发：把 `ulysses_mode=advanced_uaa` 作为 padding-mask 替代方案或性能对照。 +- 强制：advanced UAA 在 Ulysses attention 内用 variable split 处理不等 sequence/head division； + 若模型 `_sp_plan` 已 `auto_pad=True`，它收到的仍是补齐后的 evenly-divisible tensor，不会撤销 padding， + 也不会自动恢复 strict mask。两项选择必须按真实 hook→attention 顺序分别验证。 +- 禁止：因为 advanced UAA 支持 arbitrary sequence length 就推断 Flux2 不再 auto-pad，或把两条路径的 + latency/数值声明为等价。^[PR #4645] + +## SPPAD-2a — 性能数字绑定 exact padded workload，视觉附件不构成质量 gate + +- 触发：引用 skip-mask 的 latency、百分比或“无质量下降”。 +- 强制：PR 数据只绑定 vLLM 0.23.0、Omni pre-final `ad07f98b7`、Ulysses degree 8：Flux2-dev 1304²、25 steps、 + 6561→6568 tokens 为 7049.25→5444.68 ms；HunyuanVideo1.5 T2V 848×480、33 frames、30 steps、 + 14310→14312 tokens 为 16518.37→11729.65 ms。它是一次报告的 `e2e_total_ms` request timing + observation，不是重复测量或 SLA。 +- 禁止：泛化 -22.8%/-29.0% 到其他尺寸、SP degree、backend/hardware 或 merge target。附件图片/视频 + 和作者未附数字的 rebase 后复跑没有 LPIPS、PSNR、逐帧误差、重复次数或机器可审计日志，不能证明 + numerical/quality parity；默认 false 仍是明确的近似策略。 +- 验收：性能结论须在目标 SHA 固定硬件/backend、warmup/repeats 与 exact case 重跑；质量结论对 + mask true/false 使用相同 seed/input 并设模型合适的图像/视频 metric gate。^[PR #4645] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2203a9b06..5698de5f7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 06e733d3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4ab1b4ff` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5fac29b2c..65cc85094 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 06e733d3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4ab1b4ff` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6357b6ca6..e619fa421 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 06e733d3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4ab1b4ff` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 308f8dfc4..efd389075 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 06e733d3` 验证存在 +- 源码校验：以上路径均已在 `main @ 4ab1b4ff` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 67a04d817..12325e3e5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 06e733d3`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4ab1b4ff`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from af3faa6c \ - --to 06e733d3 \ + --from 06e733d3 \ + --to 4ab1b4ff \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/flux2/rules.md b/knowledge/repos/vllm-omni/models/flux2/rules.md index 6c626dd6a..e76093fce 100644 --- a/knowledge/repos/vllm-omni/models/flux2/rules.md +++ b/knowledge/repos/vllm-omni/models/flux2/rules.md @@ -1,10 +1,10 @@ --- title: "FLUX.2 规则" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5136"] +sources: ["PR #4645", "PR #5136", vllm_omni/diffusion/models/flux2/flux2_transformer.py] confidence: high --- @@ -19,6 +19,7 @@ confidence: high | FLUX.2 pipeline/transformer | FLUX2-1b | `diffusion/registry.py::_DIFFUSION_MODELS["Flux2Pipeline"]` → `diffusion/models/flux2/pipeline_flux2.py::Flux2Pipeline`、`flux2_transformer.py::Flux2Transformer2DModel` | | Mistral text encoder、FP8、component namespace | FLUX2-1a | `diffusion/models/mistral_encoder/` → component quantization selector/loader | | meta parameter、CPU offload、BF16 baseline | FLUX2-1b | FLUX.2 component loader/materialization 路径 → `Flux2Pipeline` | +| SP auto-padding、`mask_sp_padding`、dense/varlen | [SPPAD-1a](../../components/diffusion/rules-sp-padding.md#sppad-1a-padding-mask-是显式正确性性能策略不是无损优化) | Flux2 `_sp_plan` → forward context → `hidden_states_mask` → attention | FLUX.1 和 FLUX.2-Klein 不自动归到本页；必须由描述/registry key 明确命中对应 owner。 diff --git a/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md b/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md index b9b7416a6..9675a14aa 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md @@ -1,10 +1,10 @@ --- title: "HunyuanVideo-1.5（T2V/I2V,meanflow 旗标蒸馏）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/hunyuan_video/, vllm_omni/deploy/hunyuan_video_15.yaml, vllm_omni/config/pipeline_registry.py] +sources: ["PR #4645", vllm_omni/diffusion/models/hunyuan_video/, vllm_omni/deploy/hunyuan_video_15.yaml, vllm_omni/config/pipeline_registry.py] --- # HunyuanVideo-1.5 @@ -38,6 +38,9 @@ diffusion 视频,AR registry 无入口;image3 的结构见该页）。 - 依赖共享模块：[Diffusion 组件](../../components/diffusion/_index.md) （CFGParallelMixin、SP `_sp_plan`、分布式/瓦片 VAE、cache-dit adapter）、 `diffusion/models/t5_encoder.py`（glyph 路的共享 `T5EncoderModel`）。 +- SP 不可整除时仍由 `_sp_plan` auto-pad；默认 `mask_sp_padding=False` 不把尾部零 token 交给 + attention mask，以避开 varlen path，strict opt-in 与证据边界见 + [SP padding rules](../../components/diffusion/rules-sp-padding.md)。 ## 目录内容 From 5143004cb8145a5bf8992153b75b365ec554c560 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 01:52:48 +0800 Subject: [PATCH 100/465] knowledge(vllm-omni): sync e04210d6 (PR #5671) Upstream-Commit: e04210d690d619d6da34cbce3ae98e6fa31398bb Upstream-PR: #5671 Knowledge-Delta: Qwen3-Omni MUSA FP8 stage and Talker-MTP graph contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/rules.md | 10 ++++--- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 19 +++++++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/qwen-omni/_index.md | 9 ++++--- .../models/qwen-omni/architecture.md | 26 ++++++++++++++++--- 13 files changed, 66 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 84ffdf011..415f13d2e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 06e733d3cd4400bb0029f8418e2207f1441a39c0 + previous_audited_sha: 4ab1b4ff9cfc2ca233cccb5a4ba1f845288278fc audited_ref: main - audited_sha: 4ab1b4ff9cfc2ca233cccb5a4ba1f845288278fc + audited_sha: e04210d690d619d6da34cbce3ae98e6fa31398bb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5e69b5efe..e2dbee549 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4ab1b4ff9cfc2ca233cccb5a4ba1f845288278fc`. The canonical machine baseline +e04210d690d619d6da34cbce3ae98e6fa31398bb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e17972a69..ed16f18ec 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4ab1b4ff` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e04210d6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 8d1b1883b..78fbd18a2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 配置开发门禁" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-02 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py] --- # vLLM-Omni 配置开发门禁 @@ -108,8 +108,12 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR 或 `engine_extras` 各层说法不一致。 - 强制：以 `resolve_deploy_yaml → load_deploy_config → merge_pipeline_deploy → build_stage_runtime_overrides` 展开后的最终逐 stage 配置为唯一事实，逐字段打印核对。 + overlay 中的显式 `null` 也是有语义的值：例如 MUSA Qwen3-Omni profile 在 Talker 与 + Code2Wav 用 `hf_overrides.quantization_config: null` 清除 checkpoint root 的 ModelOpt + metadata，避免 BF16 audio stage 被自动识别为量化；Thinker 与非 MUSA base 保持不变。 - 禁止：拿某一层 YAML 原文当生效值；用默认值脑补缺失字段。 -- 验收：争议字段在最终逐 stage 对象中可读回，并与第一位 consumer 一致。 +- 验收：争议字段在最终逐 stage 对象中可读回，并与第一位 consumer 一致；平台 overlay + 必须同时断言受影响 stage 和未覆盖 control，不能只验证 YAML 原文。^[PR #5671] ### CONF-4a — composable strategy 只暴露已经接通的 axis diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 39cae1dff..1dc95caa8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4ab1b4ff` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e04210d6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5698de5f7..55950446e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4ab1b4ff` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e04210d6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 65cc85094..f8f3f8355 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4ab1b4ff` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e04210d6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 90ad6308f..bcfa81737 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5777", "PR #5792", "PR #5824", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -24,6 +24,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | | fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | H3 text encoder fused owner；其他模型先确认目标 main 是否已有 fused parameter | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | +| Talker-MTP FULL graph、平台 capture 能力、显式 opt-out | `mtp-graph`：`EXEC-4c` | `platforms/interface.py::supports_talker_mtp_graph_capture` → platform override → model `talker_mtp_graph_safe` → `OmniGPUModelRunner._init_talker_mtp` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -32,6 +33,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`, `EXEC-1b`, `EXEC-1c`, `EXEC-1d` | | `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`, `EXEC-2b` | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a` | +| `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | ## 严格配置校验 @@ -81,6 +83,21 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru - 验收：覆盖 blank override、多子目录 checkpoint、hook 调用顺序和 mixed-batch request-local metadata；初始化失败必须在 scheduler/worker 继续运行前暴露。 +### EXEC-4c — Talker-MTP graph 能力必须保留 tri-state 语义 + +- 触发：修改 Talker-MTP 的 FULL graph wrapper、模型 `talker_mtp_graph_safe`、平台 graph + capability，或把 talker 与其他 stage 共用同一层 wrapper。 +- 强制：只有实际 Talker stage 从平台声明模型能力；runner 读取能力时保留三态：未声明 + `None` 回退到既有 `has_separate_talker`，显式 `True` 允许 wrapper，显式 `False` 即使存在 + separate talker 也必须阻止 wrapper。上述判断只控制 dedicated Talker-MTP FULL graph + wrapper，不能跳过 MTP buffer 初始化或关闭该 stage 其余 compile/capture 路径。 +- 禁止：用 `has_separate_talker or talker_mtp_graph_safe` 吃掉显式 `False`；把 platform + 默认支持误写成所有模型未声明时一律开启；因一个算子 capture 失败而全局切 eager。 +- 验收：CPU runner 测试至少覆盖 explicit false、未声明且 separate talker、explicit true， + 以及无 separate talker 的非 Talker stage。MUSA 当前 override 为 false，因为其 stream + capture 到 `torch.multinomial` 会报 `operation not permitted when stream is capturing`； + 其他平台沿接口默认 true，但仍受模型声明与 FULL graph mode 共同约束。^[PR #5671] + ## Runner 到模型的预处理合同 - 触发条件：修改或排查 runner `_preprocess` 的逐请求 metadata 生产、phase 判定、normal/batched preprocess 选择、MTP 路由条件，或多模型共用的输入预处理合同。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e619fa421..b265a5ad1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4ab1b4ff` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e04210d6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index efd389075..128e0f521 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4ab1b4ff` 验证存在 +- 源码校验：以上路径均已在 `main @ e04210d6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 12325e3e5..dd5186ff0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4ab1b4ff`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e04210d6`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 06e733d3 \ - --to 4ab1b4ff \ + --from 4ab1b4ff \ + --to e04210d6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index 6a474fafa..4bb87699b 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -1,17 +1,17 @@ --- title: "Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS）" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-09-02 type: index tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/config/pipeline_registry.py, docs/design/qwen3_omni_tts_performance_optimization.md] +sources: ["PR #5073", "PR #5671", vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, docs/design/qwen3_omni_tts_performance_optimization.md] --- # Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS） - 常见别名：`qwen2_5_omni`、`qwen3_omni_moe`、`qwen3_tts`（家族目录：同一 thinker/ talker/code2wav 谱系的多个 checkpoint/代际，按别名规则共用本目录） -- 源码模型族（`main @ 593b4045` 验证）：`model_executor/models/qwen2_5_omni/`、 +- 源码模型族（`main @ e04210d6` 验证）：`model_executor/models/qwen2_5_omni/`、 `qwen3_omni/`、`qwen3_tts/`；pipeline registry key `qwen2_5_omni`、 `qwen2_5_omni_thinker_only`、`qwen3_omni_moe`（resolver `resolve_qwen3_omni_pipeline`，位于 `models/qwen3_omni/pipeline.py`，由 @@ -21,6 +21,9 @@ sources: ["PR #5073", vllm_omni/model_executor/models/registry.py, vllm_omni/mod （+ `_forced_aligner`、`_high_concurrency` 变体） - Qwen2.5 thinker-only 的 ModelOpt NVFP4 W4A4 checkpoint 支持、stage sub-config、Talker ingress/internal width 与硬件/accuracy 证据边界归下方 architecture 入口。 +- Qwen3-Omni 的 MUSA ModelOpt FP8 边界是 Thinker 保留量化、Talker/Code2Wav 通过 deploy + overlay 显式清除 root quantization metadata，并仅关闭不安全的 Talker-MTP FULL graph + wrapper；精确三态与验证边界归下方 architecture 入口。 - 官方历史文档：`docs/design/module/archive/async_omni_architecture.md`（以 Qwen3-Omni 为 worked example 的分层运行时快照，非 active spec）、 `docs/design/qwen3_omni_tts_performance_optimization.md`（性能优化实录） diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md index 1f0fb2e73..d19f46b2f 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md @@ -1,15 +1,15 @@ --- title: "Qwen-Omni 家族拓扑与性能结论" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-09-02 type: architecture tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/config/model.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/registry.py, vllm_omni/worker/gpu_ar_model_runner.py] +sources: ["PR #5073", "PR #5671", docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/config/model.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/registry.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/worker/gpu_model_runner.py] --- # Qwen-Omni 家族拓扑与性能结论 -以下事实在 `main @ 593b4045` 复核。 +以下事实在 `main @ e04210d6` 复核。 ## Stage 拓扑 @@ -46,6 +46,26 @@ sources: ["PR #5073", docs/design/qwen3_omni_tts_performance_optimization.md, do 更早的表来自旧 commit backport/旧依赖，不能当最终 head 证据。最终 diff 也缺 Qwen2.5 ingress≠internal 的直接 unit test。^[PR #5073] +## Qwen3-Omni 在 MUSA 上的 ModelOpt FP8 边界 + +- MUSA deploy profile 只在 stage 1 Talker 与 stage 2 Code2Wav 合并 + `hf_overrides.quantization_config: null`，显式清除 checkpoint root 的 ModelOpt metadata； + stage 0 Thinker 保留 ModelOpt FP8，非 MUSA base profile 不变。该语义位于 deploy overlay， + 没有扩张通用 arg/config resolver；合并门禁见 [CONF-3a](../../components/configuration/rules.md#conf-3a-争议以展开后的最终配置为准)。 +- Qwen3 wrapper 只在 `model_stage == "talker"` 时从当前平台写入 + `talker_mtp_graph_safe`。共享 runner 保留 tri-state：未声明时仅沿用 separate-talker legacy + 行为，false 阻止 dedicated Talker-MTP FULL graph wrapper，true 才允许；buffer 初始化及其他 + compile/capture 路径不受影响。MUSA false 的具体原因是 `torch.multinomial` 在 stream capture + 中报不允许该操作，而非整个 Qwen stage 必须 eager；共享门禁见 + [EXEC-4c](../../components/model-executor/rules.md#exec-4c-talker-mtp-graph-能力必须保留-tri-state-语义)。 +- PR 最终 diff 没有 code-predictor `forward_musa` override：review 确认当前 `CustomOp` + MUSA dispatch 已委托 `forward_cuda`，cached RoPE `_lookup` 可复用。旧 v0.24.1 集成里的 + on-the-fly native matmul workaround 不属于当前实现，不能按 PR 初版描述复活。 +- 外部证据仅是一套 2×MTT S5000、MUSA 5.2 compatibility image 上的单次 text+audio smoke： + HTTP 200、有限音频值且日志无 stage/engine failure。环境含外部兼容改动，image digest 仅记录 + 截断值，也没有 BF16 baseline、质量或性能对比，因此只能支持可运行性，不能泛化为精度/吞吐 + 结论。^[PR #5671] + ## 官方性能优化结论（docs/design/qwen3_omni_tts_performance_optimization.md） 两模型共享同一多 stage 设计，叠加同一组优化：**batching**（逐 stage 提升 GPU From 2a1642a8e0b1ec62abb9b7c62241f234954cfc45 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 02:00:52 +0800 Subject: [PATCH 101/465] knowledge(vllm-omni): sync 22dd96ec (PR #6058) Upstream-Commit: 22dd96ec5c0fa25667ef186c31e199ba7809ae50 Upstream-PR: #6058 Knowledge-Delta: MUSA platform visibility, capability, and diffusion IR priority contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-platform-runtime.md | 35 +++++++++++++++++++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/architecture.md | 16 ++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 11 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 415f13d2e..d0bc3858c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4ab1b4ff9cfc2ca233cccb5a4ba1f845288278fc + previous_audited_sha: e04210d690d619d6da34cbce3ae98e6fa31398bb audited_ref: main - audited_sha: e04210d690d619d6da34cbce3ae98e6fa31398bb + audited_sha: 22dd96ec5c0fa25667ef186c31e199ba7809ae50 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e2dbee549..74f084dd1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e04210d690d619d6da34cbce3ae98e6fa31398bb`. The canonical machine baseline +22dd96ec5c0fa25667ef186c31e199ba7809ae50`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ed16f18ec..2aaca8d70 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e04210d6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 22dd96ec` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1dc95caa8..4a1bc7685 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e04210d6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 22dd96ec` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -30,6 +30,7 @@ sources: [] |---|---| | 理解共享职责和数据流 | [architecture](architecture.md) | | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | +| 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md new file mode 100644 index 000000000..a5eed3dc0 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md @@ -0,0 +1,35 @@ +--- +title: "Diffusion 平台运行时规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #6058", vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/platforms/musa/platform.py] +confidence: high +--- + +# Diffusion 平台运行时规则 + +本页承载 diffusion worker 与硬件 platform 之间的配置合同；模型自己的 kernel、数值与性能 +结论仍归共享 layer/backend 或模型 owner。 + +## Direct 代码快速入口 + +| PR 描述在做什么 | 规则 | 第一批 live 源码 | +|---|---|---| +| IR-op priority、Inductor/eager 默认顺序、模型 hook 覆盖 | `DIFFPLAT-1a` | platform `get_default_ir_op_priority` → `diffusion_worker._resolve_ir_op_priority` → model `get_diffusion_ir_op_priority_func` | + +## DIFFPLAT-1a — platform IR-op priority 必须区分 Inductor 与 eager + +- 触发：平台新增或修改 `get_default_ir_op_priority()`，或 diffusion worker 改变 platform + default 与模型 hook 的合并顺序。 +- 强制：worker 先取当前 platform default，再让模型级 `get_diffusion_ir_op_priority_func()` + 做精确覆盖。MUSA 与 ROCm 的通用基线是：backend 为 `inductor` 且 compilation mode 非 + `NONE` 时只优先 `native`，否则按 `vllm_c → native`；返回值必须经 + `IrOpPriorityConfig.with_default()` 构造，使显式 per-op priority 仍可表达。 +- 禁止：在 compile 路径优先可能产生不兼容 custom-call 的 `vllm_c`；把 CUDA 固定优先级、 + ROCm 的 AITER/RMSNorm 特例或单模型 override 泛化到 MUSA；仅凭实现与 ROCm 排序一致声称 + MUSA kernel 已验证。 +- 验收：平台单测至少覆盖 Inductor enabled、mode `NONE`、非 Inductor backend，并由 worker + 测试断言 platform default 先于 model hook。PR #6058 没有新增自动测试或 MUSA runtime + 运行，当前证据只证明静态实现与现有 worker consumer 接通。^[PR #6058] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 55950446e..d3c04b3fe 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e04210d6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 22dd96ec` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f8f3f8355..24129be17 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e04210d6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 22dd96ec` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/architecture.md b/knowledge/repos/vllm-omni/components/model-executor/architecture.md index 588d1261b..d9bffa75d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/architecture.md +++ b/knowledge/repos/vllm-omni/components/model-executor/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: architecture tags: [vllm-omni, components, model-executor] -sources: ["PR #5610", "PR #5744", docs/design/feature/omni_async_output_materialization.md, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, tests/worker/test_omni_connector_mixin.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] +sources: ["PR #5610", "PR #5744", "PR #6058", docs/design/feature/omni_async_output_materialization.md, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/stage_utils.py, tests/worker/test_omni_connector_mixin.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] --- # Model Executor 共享架构 @@ -35,6 +35,20 @@ sources: ["PR #5610", "PR #5744", docs/design/feature/omni_async_output_material 启动类错误优先沿“最终 stage 配置 → runtime devices → 启动前校验 → worker rank”读取这四段。只有其中一段把错误状态交给其他模块时才横向展开。 +## MUSA 平台接口边界 + +`MUSAOmniPlatform` 必须把共享 stage/runtime 接口落到 MUSA 自己的 runtime，而不是借用 +CUDA 名称：设备能力来自 `torch.musa.get_device_capability()`；stage visibility 的 set/unset +成对操作 `MUSA_VISIBLE_DEVICES`。后者被 `entrypoints/stage_utils.py` 的逻辑→物理设备映射和 +`stage_engine_startup.py` 的 scoped subprocess spawn 直接消费，因此既要支持设置，也要在异常 +路径恢复原值或删除变量。传入值在调用链中已归一化为 `str`；平台签名虽沿共享接口保留 +`str | int | None`，不能据此声称任意 `None` 可直接写入 `os.environ`。 + +本次只做平台实现对齐，没有新增 MUSA runtime/unit 测试；PR 作者仅运行 compileall、 +pre-commit 与 diff check，并明确本地缺少 torch/vLLM/vllm_musa。设备 capability 查询、真实 spawn 隔离和 +capability 驱动的 FlashAttention 选择因此仍需 MUSA 环境验证。diffusion worker 的 IR-op +priority 合同见 [DIFFPLAT-1a](../diffusion/rules-platform-runtime.md#diffplat-1a-platform-ir-op-priority-必须区分-inductor-与-eager)。^[PR #6058] + ## AR async Omni output materialization `GPUARModelRunner` 可把 `OmniModelRunnerOutput` 的 D2H、逐请求切片和 payload 构造从 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b265a5ad1..57dc4c267 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e04210d6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 22dd96ec` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 128e0f521..cef170859 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e04210d6` 验证存在 +- 源码校验：以上路径均已在 `main @ 22dd96ec` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index dd5186ff0..f0c46e62c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e04210d6`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 22dd96ec`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 4ab1b4ff \ - --to e04210d6 \ + --from e04210d6 \ + --to 22dd96ec \ --repo \ --mode report-only ``` From 1c2c56da0242d5a31597c56f1415b19ecada0e33 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 02:07:47 +0800 Subject: [PATCH 102/465] knowledge(vllm-omni): sync 348cf95a (PR #6056) Upstream-Commit: 348cf95a895061a403ce3b611b45b36fb24ebc61 Upstream-PR: #6056 Knowledge-Delta: MiniCPM-o 4.5 online-serving CI and duplex fixture contracts --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 7 ++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minicpm-o-4-5/_index.md | 5 +++- .../vllm-omni/models/minicpm-o-4-5/rules.md | 27 +++++++++++++++++-- 12 files changed, 47 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d0bc3858c..466edf637 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e04210d690d619d6da34cbce3ae98e6fa31398bb + previous_audited_sha: 22dd96ec5c0fa25667ef186c31e199ba7809ae50 audited_ref: main - audited_sha: 22dd96ec5c0fa25667ef186c31e199ba7809ae50 + audited_sha: 348cf95a895061a403ce3b611b45b36fb24ebc61 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 74f084dd1..9ceecbe9d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -22dd96ec5c0fa25667ef186c31e199ba7809ae50`. The canonical machine baseline +348cf95a895061a403ce3b611b45b36fb24ebc61`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 431b83d7a..f3c0ce0aa 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -41,6 +41,11 @@ confidence: high ^[PR #5310] ^[PR #5524] ^[PR #5543] ^[PR #5780] ^[PR #5872] ^[PR #6048] ^[PR #6096] ^[PR #6102] +MiniCPM-o online suite 的具体矩阵必须从 server args 读取：core 只跑 async chunk，expansion +分别跑 sync/async，duplex 又有自己的共享 server fixture。marker 与 parametrization 只能证明 +collection 意图；没有实际 runtime 结果时，不能证明目标硬件行为。模型专属 fixture 合同见 +[MCPMO-5b](../models/minicpm-o-4-5/rules.md#mcpmo-5b-online-serving-ci-必须显式区分-chunk-模式与-duplex-fixture-语义)。^[PR #6056] + ## OMNI-CI-1b — 回归 fence 必须锚定可观察合同且非空转 - 触发：新增 bug regression、API guard、route census 或 middleware fence。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2aaca8d70..fb89fa6c7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 22dd96ec` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 348cf95a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 4a1bc7685..d2a6adcc8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 22dd96ec` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 348cf95a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d3c04b3fe..faeb192ee 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 22dd96ec` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 348cf95a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 24129be17..8b5807dcf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 22dd96ec` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 348cf95a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 57dc4c267..528914dcf 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 22dd96ec` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 348cf95a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cef170859..3df88d072 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 22dd96ec` 验证存在 +- 源码校验：以上路径均已在 `main @ 348cf95a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f0c46e62c..9387c5b57 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 22dd96ec`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 348cf95a`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from e04210d6 \ - --to 22dd96ec \ + --from 22dd96ec \ + --to 348cf95a \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index a89da2901..4360eda97 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py] +sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py] confidence: high --- @@ -46,6 +46,9 @@ Talker 只收集 sampling-eligible active rows 批量执行 codec projection 与 Daily-Omni、Seed-TTS simplex/realtime duplex 已进入 accuracy/perf harness；阈值、逐轮 metrics、 单卡 H100 perf 与“baseline 不是 regression gate”的证据边界见 MCPMO-5a。 +Online serving CI 的 core suite 固定 async chunk，expansion 分开覆盖 sync/async，native duplex +fixture 不再覆盖 eager/KV cap 并固定 active-speech turn window；验证证据边界见 MCPMO-5b。 + 描述直达源码与模型专有门禁见 [rules](rules.md#direct-代码快速入口)；新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 731a0f4c9..82e977e5d 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] confidence: high --- @@ -28,6 +28,7 @@ confidence: high | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | | instructions/persona/voice/mode update、prefill slots、context lock | MCPMO-4b | `experimental/fullduplex/minicpmo45/session.py` → runtime adapter/session runner | | Daily-Omni/Seed-TTS accuracy、simplex/duplex perf、TTFT/TTFP/RTF | MCPMO-5a | `tests/e2e/accuracy/minicpmo_4_5/`、`tests/dfx/perf/tests/test_minicpmo_4_5*.json` → `run_benchmark.py` | +| online serving CI、async/sync chunk matrix、duplex fixture、local ref audio | MCPMO-5b | `tests/e2e/online_serving/test_minicpmo_4_5*.py` → `helpers/minicpmo_4_5_duplex.py` | ## MCPMO-1a — trust_remote_code 服从用户选择 @@ -188,7 +189,9 @@ confidence: high - 禁止：用 server VAD cancellation 冒充 native interruption；每轮重建 Stage0；让 LISTEN 因 无 Talker payload 失败。 - 验收：native case 完成响应≥2、cancel/truncate=0、listen≥1；opt-in hard interrupt 恰好一次 - terminal、fence 后无 stale delta、保留打断语句且后续响应成功。 ^[PR #6154] ^[PR #6170] + terminal、fence 后无 stale delta、保留打断语句且后续响应成功。response-required fixture 若要求 + 每轮 SPEAK，应重放已验证的完整 active-speech window；任意较短 mid-utterance slice 合法返回 + LISTEN，不能把它当模型失败或修改生产决策阈值。 ^[PR #6056] ^[PR #6154] ^[PR #6170] ## MCPMO-4b — session update 与派生 reservation 原子提交 @@ -224,5 +227,25 @@ confidence: high 未提交 patch，证据脚本还吞掉 step failure，且把版本号写作 commit。因而不能从这些数字声称 merge target 的可复现质量/性能通过。^[PR #5524] +## MCPMO-5b — online serving CI 必须显式区分 chunk 模式与 duplex fixture 语义 + +- 触发：修改 MiniCPM-o 4.5 online serving、expansion、native duplex 测试或其 deploy helper。 +- 强制：core online suite 显式启动 `--async-chunk`，并把 bounded sampling 保持在 fixture-local + 派生配置；expansion suite 以两个独立 parametrization 覆盖 `--no-async-chunk` 与 + `--async-chunk`。不得用 deploy 默认值或参数 ID 猜实际模式。audio→text+audio case 若要稳定 + 验证 TTS，必须显式请求 text/audio modalities、`use_tts_template=True`、 + `enable_thinking=False` 并断言目标关键词。 +- duplex fixture 保留 Talker `max_model_len=4096`，但不覆盖三个 stage 的 KV sizing，不另造 eager-only + server；protocol smoke 与 response tests 使用同一 server config。response-required 多轮测试每轮 + 重放同一 active-speech window，避免较短的 mid-utterance slice 合法地产生 LISTEN。reference audio + 先接受本地 checkpoint 目录（包括绝对 `MODEL` 路径），不存在时才从本地 HF cache 解析。 +- 禁止：把删除测试专用 KV cap 解释成生产 deploy 资源变更；把移除 assertion 的 JSON failure + message 解释成行为放宽；或在输入与 oracle 已改变后声称复现并修复原始 mixed-input + audio-similarity 故障。 +- 验收：至少收集 core async、expansion sync/async、duplex protocol 与 advanced session cases， + 并记录实际执行结果及硬件；local checkpoint 与 cache fallback 各一例。changed tests、marker + 或 parametrization 只能证明 collection 意图；没有实际 runtime 结果时，不能声称 CUDA/NPU、 + 音频质量或 flake 已闭环。^[PR #6056] + 共享 bridge/batch 规则见 [Model Executor rules](../../components/model-executor/rules.md)； 公开入口完整性见 [model adaptation guardrails](../../review/guides/model-adaptation-guardrails.md)。 From bf2855a46d3f56a286143ce4afbd43fcbf6a060a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 02:20:04 +0800 Subject: [PATCH 103/465] knowledge(vllm-omni): sync 89015d5c (PR #2783) Upstream-Commit: 89015d5cad0a55dc970fdde1badaf74d39b8c4f7 Upstream-PR: #2783 Knowledge-Delta: diffusion distilled LoRA lifecycle, fusion, and model scope --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../components/diffusion/rules-lora.md | 84 +++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +- .../vllm-omni/models/qwen-image/_index.md | 7 +- .../repos/vllm-omni/models/wan2-2/_index.md | 8 +- .../vllm-omni/models/wan2-2/architecture.md | 8 +- 14 files changed, 115 insertions(+), 19 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-lora.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 466edf637..d7b11d9fc 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 22dd96ec5c0fa25667ef186c31e199ba7809ae50 + previous_audited_sha: 348cf95a895061a403ce3b611b45b36fb24ebc61 audited_ref: main - audited_sha: 348cf95a895061a403ce3b611b45b36fb24ebc61 + audited_sha: 89015d5cad0a55dc970fdde1badaf74d39b8c4f7 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9ceecbe9d..1546e938b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -348cf95a895061a403ce3b611b45b36fb24ebc61`. The canonical machine baseline +89015d5cad0a55dc970fdde1badaf74d39b8c4f7`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fb89fa6c7..a583aa205 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 348cf95a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 89015d5c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d2a6adcc8..df5085220 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 348cf95a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 89015d5c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -32,6 +32,7 @@ sources: [] | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | +| PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | | video/audio mux、DLO DP wave、result queue、async pump、SHM ownership 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md new file mode 100644 index 000000000..a74f02356 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -0,0 +1,84 @@ +--- +title: "Diffusion LoRA 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/entrypoints/test_async_omni_diffusion_config.py] +confidence: high +--- + +# Diffusion LoRA 规则 + +本页区分 request-time PEFT adapter 与 startup-time distilled weight fusion。跨 component identity +仍服从 [DIFF-2f/2g](rules-component-lifecycle.md)，模型专属 transformer 数量与顺序留在模型 owner。 + +## Direct 代码快速入口 + +| PR 描述在做什么 | 规则 | 第一批 live 源码 | +|---|---|---| +| `--lora-backend`、startup path/scale、deploy/CLI projection | `DIFF-2l` | `entrypoints/cli/serve.py` → `async_omni_engine.py`/stage config → `OmniDiffusionConfig` → `DiffusionWorker.init_lora_manager` | +| distilled merge、key conversion、packed QKV、alpha、unload | `DIFF-2m` | `diffusion/lora/loader.py` → pipeline mixin → transformer parameter | +| Wan dual transformer、high/low-noise file order、partial load | `DIFF-2n` | `WanLoraLoaderMixin` → `get_transformer_from_pipeline(name)` → `transformer`/`transformer_2` | + +| 审查组 | 什么时候触发 | 规则 ID | +|---|---|---| +| `checkpoint-distributed` | diffusion LoRA backend、checkpoint、融合与 transformer mapping | `DIFF-2l`, `DIFF-2m`, `DIFF-2n` | + +## DIFF-2l — backend 是 startup 配置，不是 request 字段 + +- 触发：增加 LoRA backend、CLI/deploy 字段、worker 初始化或 request adapter 路由。 +- 强制：`peft` 保持默认，使用 `DiffusionLoRAManager` 的 cache 与 per-request `LoRARequest`； + `distill` 必须同时有 startup `lora_path`，在模型加载后只调用一次 pipeline + `load_lora_weights()`，融合结果在 server 生命周期内持续生效，普通请求不再携带 adapter。 + CLI → default/structured stage → diffusion projection → worker 必须保留 backend、path 与 scale； + stage 显式值优先于全局注入。 +- 禁止：把 `lora_backend` 写入 PEFT request 的 dead `extra_args`；让 serve CLI 接受多个 PEFT startup + path；把 distill 描述成可逐请求切换或 LRU cache。pipeline 没有 `load_lora_weights` 时当前只 + warning 并以未修改 base weights 继续，是 fail-open capability gap，不能声称任意 diffusion + pipeline 已支持。直接 config 仍可能把多 path 送到只接受单 path 的 PEFT manager，CLI 校验 + 不能冒充所有入口的通用门禁。 +- 验收：CLI 正负测试覆盖 distill 无 path、PEFT 多 path、单/多 path 投影与 stage precedence； + worker 测试分别断言 PEFT manager、distill 单次融合、unsupported warning。`lora_scale` 对 + distill 完全不应用运行时 scale：大于 1 只 warning，小于等于 1 静默忽略；不得声称 scale 对 + fusion 生效。config 类型同时出现 scalar/list 表述，扩展 list scale 前必须先统一 projection + 与 worker 比较语义。distill caller 还必须传具体 `.safetensors` 文件；目录或 Hub repo 没有 + `weights_name`，当前调用链无法解析。^[PR #2783] + +## DIFF-2m — distilled delta 必须按真实 parameter 布局可逆合并 + +- 触发：修改 checkpoint key conversion、LoRA A/B/bias、packed parameter 或 load/unload。 +- 强制：delta 为 `B @ A`，转到目标 parameter 的 device/dtype 后在 `no_grad` 下原位加；unload + 必须用同一 state dict、key remap 与布局原位减回。`stacked_params_mapping` 的每个 source shard + 都必须存在后才按目标 packed 顺序 concat，不能把缺 shard 当零。key remap 依声明顺序累积应用。 +- Qwen 已是 Diffusers `lora_A/lora_B` 且带 `.alpha` 时，先把 `alpha/rank` 折进 B 并移除 alpha； + alpha 缺 A/B 必须失败。non-Diffusers key 或其余带 alpha 格式再走 pipeline MRO 对应 converter； + `.to_out.0` 等名称只在转换后映到树内参数。 +- 禁止：静默丢弃非单位 alpha；只验证已消费 key 数却不比较恢复后的 base weight；把 warning 的 + unmatched keys 当严格成功。当前 loader 对 partial/zero-match key 都只 warning 并继续，是 + fail-open gap，审查时必须核对目标模块集合和实际 applied-key count。 +- 兼容边界：融合发生在 `load_model()` 后，对 dense parameter 直接 `param.add_(B @ A)`；目标没有 + TP/HSDP shard-aware delta，也没有量化 stored weight/scale 的合法 mutation 合同。因此当前可证明 + 的 correctness envelope 仅为 dense、unquantized、TP=1；其他组合在宣称支持前必须增加 fail-closed + compatibility check 或对应 shard/quantization-aware 实现。 +- 验收：CPU 测试覆盖普通/bias/packed load、unload 精确恢复、累积 remap、non-unit alpha、缺配对 + alpha 与 unknown key；真实 smoke 另需固定 adapter、模型 revision、sampling 参数与输出比较。 + 当前 CPU evidence 使用 TP=1 `nn.Linear`；图片/视频附件是人工示例，没有 TP/HSDP/量化、 + 自动质量阈值或性能测量，不能作为持续 correctness gate。^[PR #2783] + +## DIFF-2n — Wan 多文件必须逐位置绑定 architecture-declared transformer + +- 触发：Wan 单/双 transformer pipeline 加载 distilled LoRA，或 boundary 部分加载。 +- 强制：单 transformer architecture 只接受一个具体 `.safetensors`；`has_transformer_2` 声明的 + 双 transformer architecture 必须按 + `[transformer, transformer_2]` 位置提供等长文件列表（LightX2V 示例即 high-noise 在前、 + low-noise 在后）。getter 必须接收目标 attribute 名，不能每次都返回第一 transformer。 + 即使 boundary 使其中一个 target 未实例化，仍要求两个 path；未加载 target 记录 warning 并跳过， + 已加载模块仍保存各自 state dict 供 unload。 +- 禁止：用远端目录加硬编码文件名猜 checkpoint；把两个文件都融合到 `transformer`；从 T2V/I2V + mixin 接入外推 VACE/S2V 或所有 registry Wan 变体支持。loader 只信任 caller 顺序，无法从文件 + 内容结构性识别 high/low 是否颠倒。 +- 验收：测试用不同 delta 断言两个 transformer 分别变化并在 unload 后恢复；覆盖文件数不匹配、 + 单 transformer、多 transformer与缺失 target；high/low 顺序必须另用固定 checkpoint 的模型级 + 语义/质量证据验证。目标单测覆盖双 transformer 的 load/unload，但没有真实 Wan checkpoint + 数值或视频质量自动 gate。^[PR #2783] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index f74a41bf9..52669d76f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -37,7 +37,7 @@ confidence: high |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1j` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2k` | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2n` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index faeb192ee..29011ec21 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 348cf95a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 89015d5c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8b5807dcf..e71f32914 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 348cf95a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 89015d5c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 528914dcf..40a300350 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 348cf95a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 89015d5c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3df88d072..71e3c5587 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 348cf95a` 验证存在 +- 源码校验：以上路径均已在 `main @ 89015d5c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9387c5b57..5df069e09 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 348cf95a`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 89015d5c`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 22dd96ec \ - --to 348cf95a \ + --from 348cf95a \ + --to 89015d5c \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-image/_index.md b/knowledge/repos/vllm-omni/models/qwen-image/_index.md index 93132f896..d8a7bbb00 100644 --- a/knowledge/repos/vllm-omni/models/qwen-image/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-image/_index.md @@ -1,10 +1,10 @@ --- title: "Qwen-Image（base/Edit/Edit-Plus/Layered/DMD2 五变体）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/qwen_image/, vllm_omni/diffusion/registry.py] +sources: ["PR #2783", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/qwen_image/, vllm_omni/diffusion/registry.py, tests/diffusion/lora/test_loader.py] --- # Qwen-Image @@ -39,6 +39,9 @@ sources: [vllm_omni/diffusion/models/qwen_image/, vllm_omni/diffusion/registry.p pipeline 共用）;DMD2 经 `DMD2PipelineMixin` （`pipeline_qwen_image.py:1097`）;prompt 长度校验 `validate_prompt_sequence_lengths`。 +- distilled LoRA 当前由 `QwenImagePipeline` 及继承它的 DMD2 变体接入 `QwenImageLoraLoaderMixin`；支持 key converter、 + packed parameter 与 Diffusers non-unit alpha/rank 缩放。共享 load/unload 门禁见 + [DIFF-2m](../../components/diffusion/rules-lora.md#diff-2m-distilled-delta-必须按真实-parameter-布局可逆合并)；Edit/EditPlus/Layered 未接入，converter map 有名称也不代表 pipeline 拥有 loader mixin。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/wan2-2/_index.md b/knowledge/repos/vllm-omni/models/wan2-2/_index.md index 8b862ea64..7914edbbe 100644 --- a/knowledge/repos/vllm-omni/models/wan2-2/_index.md +++ b/knowledge/repos/vllm-omni/models/wan2-2/_index.md @@ -1,10 +1,10 @@ --- title: "Wan 2.2（六架构视频家族:T2V/I2V/VACE/S2V/DMD2×2）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/wan2_2/, vllm_omni/deploy/wan2_2_ti2v.yaml, vllm_omni/diffusion/models/dmd2/mixin.py] +sources: ["PR #2783", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/wan2_2/, vllm_omni/deploy/wan2_2_ti2v.yaml, vllm_omni/diffusion/models/dmd2/mixin.py, tests/diffusion/lora/test_loader.py] --- # Wan 2.2 @@ -61,6 +61,10 @@ sources: [vllm_omni/diffusion/models/wan2_2/, vllm_omni/deploy/wan2_2_ti2v.yaml, transformer)——省显存开关,评审时别当普通超参改。 - S2V 无捆绑 deploy YAML（flow_shift 3.0 只在代码里）——部署设置 pin 上无 文档。 +- distilled LoRA 只在 `Wan22Pipeline` 与 `Wan22I2VPipeline` 显式接入；双专家按位置把 high/low-noise + 两个具体文件分别融合进 `transformer`/`transformer_2`；DMD2 子类继承该能力，VACE/S2V 不继承。 + 双专家即使因 boundary 少加载一个 target 仍要求两个 path，未加载者 warning 后跳过；精确门禁见 + [DIFF-2n](../../components/diffusion/rules-lora.md#diff-2n-wan-多文件必须逐位置绑定-architecture-declared-transformer)。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/wan2-2/architecture.md b/knowledge/repos/vllm-omni/models/wan2-2/architecture.md index f719c6b5a..371d57dd0 100644 --- a/knowledge/repos/vllm-omni/models/wan2-2/architecture.md +++ b/knowledge/repos/vllm-omni/models/wan2-2/architecture.md @@ -1,10 +1,10 @@ --- title: "Wan 2.2 架构" created: 2026-07-21 -updated: 2026-08-10 +updated: 2026-09-02 type: architecture tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py, vllm_omni/diffusion/models/dmd2/mixin.py, "PR #5969"] +sources: ["PR #2783", "PR #5969", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py, vllm_omni/diffusion/models/dmd2/mixin.py] --- # Wan 2.2 架构 @@ -44,6 +44,10 @@ diffusion 设施见 [Diffusion 组件](../../components/diffusion/_index.md)。 extra_args 里剥掉 `sample_solver`/`flow_shift`**——防止基类把 scheduler 换回去。给 DMD2 变体"加回 CFG/负向"的 PR 违反此合同。 - 量化:每 transformer 独立解析 `quantization_config`,双专家可各自量化。 +- distilled LoRA 同样按 architecture-declared transformer ownership 分开：T2V/I2V 及其 DMD2 + 子类的 mixin 读取 `has_transformer_2`，要求位置 0/1 分别对应 `transformer`/`transformer_2`， + 并通过具名 getter 避免两个 delta 都落到第一 transformer。boundary 即使少加载一个 target 也不 + 减少 path cardinality；该 startup fusion 是 server-lifetime 状态，不改变请求协议，VACE/S2V 未接入。 ## 从输入到输出的主要流程 From 8d15b0c12cf0505dae7b9658d8c6ef8672216c4e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 02:27:00 +0800 Subject: [PATCH 104/465] knowledge(vllm-omni): sync 584d78c6 (PR #5823) Upstream-Commit: 584d78c673fe7836809c2a197c3f4843e2fd9ac4 Upstream-PR: #5823 Knowledge-Delta: Flux Kontext tiny-model CI coverage ownership --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 22 ++++++++++++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../repos/vllm-omni/models/flux/_index.md | 8 +++++-- 11 files changed, 39 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d7b11d9fc..f2284ff04 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 348cf95a895061a403ce3b611b45b36fb24ebc61 + previous_audited_sha: 89015d5cad0a55dc970fdde1badaf74d39b8c4f7 audited_ref: main - audited_sha: 89015d5cad0a55dc970fdde1badaf74d39b8c4f7 + audited_sha: 584d78c673fe7836809c2a197c3f4843e2fd9ac4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1546e938b..52b125a08 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -89015d5cad0a55dc970fdde1badaf74d39b8c4f7`. The canonical machine baseline +584d78c673fe7836809c2a197c3f4843e2fd9ac4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index f3c0ce0aa..d590906a1 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5836", "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -21,6 +21,7 @@ confidence: high | pre-commit、SPDX、shellcheck、stability marker | `OMNI-CI-2a` | `.pre-commit-config.yaml`、`.buildkite/**`、`tools/**` | | xdist、共享 worker、下载 fixture、进程池 | `OMNI-CI-2b` | `tests/conftest.py`、`tests/helpers/**`、`tests/model_tests/**` | | 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | +| diffusion tiny builder、model settings、alignment exclusion、重模型 OOM | `OMNI-CI-2d` | `tests/model_tests/diffusion/{diff_model_builders,model_settings,test_alignment}.py` → common offline tests | | perf baseline、hardware label、DFX result artifact、assert-baseline | `OMNI-CI-3a` | `tests/dfx/conftest.py`、`tests/dfx/perf/scripts/run_benchmark.py`、`run_diffusion_benchmark.py`、`tests/dfx/perf/tests/**` | ## OMNI-CI-1a — 硬件 lane 必须真实收集并执行目标路径 @@ -105,6 +106,25 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 开始；level-2 terminal case 之后不再复用 server；统计目标 lane 确实只初始化预期数量的 engine，且 cleanup 后无 worker/device state 遗留。^[PR #5713] +## OMNI-CI-2d — diffusion tiny builder 必须替代资源缩减而不是能力缩减 + +- 触发：重模型 online E2E 因显存 OOM 迁移到 tiny-model framework，或修改 + `DIFFUSION_TEST_SETTINGS` / `EXCLUDED_MODELS`。 +- 强制：builder 从真实 pipeline/config 构造缩小 checkpoint，按组件分别收缩 encoder/DiT； + attention head dim 改变时同比缩放 `axes_dims_rope`，并保留 architecture 要求的轴数与总维度。 + settings 必须列出真实 supported tasks 和每个 acceleration group；只有 builder/settings 已接通 + common tests 后，才从 alignment exclusion 删除模型。 +- 禁止：为消除 OOM 直接删除 TP/CFG/cache/offload coverage；把 online suite 从双卡 parallel cases + 缩为单卡 base 后仍声称 online lane 覆盖这些组合；从 tiny checkpoint 通过外推 full checkpoint + 显存、数值质量或吞吐。 +- 验收：alignment 双向断言 registry 中每个非排除 pipeline 有 settings，且 settings 不指向排除项； + common offline suite 对每个 task 跑 base，并逐组执行声明的 acceleration。Flux Kontext 当前 tiny + contract 是 text-to-image + image-to-image，附加 TP+CPU offload、CFG+CPU offload、以及 + TP+CPU offload+Cache-DiT；online expansion 只保留单 L4 base smoke，parallel coverage 的所有权迁到 + common offline suite，不代表 full-model OOM 已被修复。tiny builder 从收缩后的真实组件配置初始化 + random BF16 weights，只证明 construction、routing 与 control-flow，不证明真实 checkpoint 加载、 + 输出质量、显存或吞吐。^[PR #5823] + ## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 - 触发：修改 perf JSON 的 `baseline`、硬件 label/marker、benchmark result schema，或恢复性能阈值断言。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a583aa205..ce9d31467 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 89015d5c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 584d78c6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index df5085220..bdd13b91e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 89015d5c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 584d78c6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 29011ec21..bdefaafec 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 89015d5c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 584d78c6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e71f32914..f26c31eaf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 89015d5c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 584d78c6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 40a300350..bae707fd9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 89015d5c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 584d78c6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 71e3c5587..2c245c883 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 89015d5c` 验证存在 +- 源码校验：以上路径均已在 `main @ 584d78c6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5df069e09..167c7ba6f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 89015d5c`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 584d78c6`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 348cf95a \ - --to 89015d5c \ + --from 89015d5c \ + --to 584d78c6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/flux/_index.md b/knowledge/repos/vllm-omni/models/flux/_index.md index 3d76b572b..e84a21998 100644 --- a/knowledge/repos/vllm-omni/models/flux/_index.md +++ b/knowledge/repos/vllm-omni/models/flux/_index.md @@ -1,10 +1,10 @@ --- title: "FLUX.1（base / Kontext 编辑 / DMD2 蒸馏）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/flux/, vllm_omni/diffusion/registry.py] +sources: ["PR #5823", tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, vllm_omni/diffusion/models/flux/, vllm_omni/diffusion/registry.py] --- # FLUX.1 @@ -37,6 +37,10 @@ sources: [vllm_omni/diffusion/models/flux/, vllm_omni/diffusion/registry.py] （`pipeline_flux.py:685`,FastGen DMD2 蒸馏少步数变体）。 - CFG-parallel 有 `check_cfg_parallel_validity` 对 `get_classifier_free_guidance_world_size()` 的校验。 +- Flux Kontext 已从 tiny-model alignment exclusion 移除：builder 共用 Flux 的 CLIP/T5/DiT shrink， + common suite 覆盖 text-to-image、image-to-image，以及 TP/CFG/CPU-offload/Cache-DiT 三组组合； + 原 online expansion 相应只保留单 L4 base smoke。该变化是测试资源/覆盖归属，不改变生产 pipeline； + 验收门禁见 [OMNI-CI-2d](../../ci/rules.md#omni-ci-2d-diffusion-tiny-builder-必须替代资源缩减而不是能力缩减)。 ## 什么时候查这里 From accf380e9bc48325876e04b89f2e25e7d63a65e2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 02:46:17 +0800 Subject: [PATCH 105/465] knowledge(vllm-omni): sync 2ce15ab7 (PR #5976) Upstream-Commit: 2ce15ab7a023d4fb800629e36c06bbe75a4324e2 Upstream-PR: #5976 Knowledge-Delta: vLLM 0.27 runtime compatibility, scheduler, serving, diffusion, CI, and model contracts --- adapters/vllm_omni/manifest.yaml | 8 +++++ adapters/vllm_omni/release_baseline.yaml | 5 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 24 +++++++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-upstream-compat.md | 33 +++++++++++++++++++ .../components/distributed/_index.md | 2 +- .../vllm-omni/components/distributed/rules.md | 12 ++++++- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 25 +++++++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 13 +++++++- .../vllm-omni/components/serving/_index.md | 3 +- .../serving/rules-upstream-compat.md | 23 +++++++++++++ knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/lingbot-video/rules.md | 10 ++++-- .../vllm-omni/models/qwen-omni/_index.md | 6 +++- knowledge/repos/vllm-omni/tooling/rules.md | 13 +++++++- 19 files changed, 174 insertions(+), 20 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index d530dd2bd..cc28a0afe 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -143,6 +143,11 @@ modules: quantization: local_paths: [vllm_omni/quantization/] wave: 1 + runtime_patch: + local_paths: [vllm_omni/patch.py] + test_paths: [tests/diffusion/test_inductor_divisibility_patch.py] + wave: 1 + risk: high sampling: local_paths: [vllm_omni/sample/] wave: 1 @@ -160,6 +165,9 @@ modules: wave: 2 review_routes: +- prefix: vllm_omni/patch.py + owner: tooling + doc: repos/vllm-omni/tooling/rules.md - prefix: vllm_omni/config/ owner: config doc: repos/vllm-omni/components/configuration/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f2284ff04..dccc93d11 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 89015d5cad0a55dc970fdde1badaf74d39b8c4f7 + previous_audited_sha: 584d78c673fe7836809c2a197c3f4843e2fd9ac4 audited_ref: main - audited_sha: 584d78c673fe7836809c2a197c3f4843e2fd9ac4 + audited_sha: 2ce15ab7a023d4fb800629e36c06bbe75a4324e2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -78,6 +78,7 @@ path_owners: runtime-core: - vllm_omni/__init__.py - vllm_omni/data_entry_keys.py + - vllm_omni/patch.py - vllm_omni/request.py # Direct review destination for every structural owner above. The audit checks diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 52b125a08..89a5b3392 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -584d78c673fe7836809c2a197c3f4843e2fd9ac4`. The canonical machine baseline +2ce15ab7a023d4fb800629e36c06bbe75a4324e2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index d590906a1..3f4d1ef72 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5976", docker/Dockerfile.ci, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -125,6 +125,17 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 random BF16 weights，只证明 construction、routing 与 control-flow，不证明真实 checkpoint 加载、 输出质量、显存或吞吐。^[PR #5823] +## OMNI-CI-2e — release rebase 的 image、容器资源与实际辅助负载必须成套对齐 + +- 触发：升级 vLLM/torch base image，或新版本在 startup 增加资源检查、编译缓存与 warmup 行为。 +- 强制：正式 release 直接使用同版本 base image 的自洽依赖集；只有目标是未发布 SHA 时才恢复 + wheel 重装及其 torch/CUDA/NumPy ABI 修复。容器显式满足新 startup check（共享内存等），硬件数 + 同时计入测试辅助模型：主模型占卡时，Whisper 等 grader 需要 spare accelerator 才能避免 CPU 超时。 +- 禁止：在 release image 上保留补偿旧 image 的依赖手术；用 Docker 默认 `/dev/shm`；只按被测模型 + 卡数配置 lane 而忽略 grader；把部分 nightly green 当全部兼容问题已关闭。 +- 验收：image tag 与目标 release 一致；startup 通过共享内存 preflight；目标 lane 证明实际 grader + device；每个残余失败独立记录，尤其数值/质量回归不能由 API 兼容测试替代。^[PR #5976] + ## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 - 触发：修改 perf JSON 的 `baseline`、硬件 label/marker、benchmark result schema，或恢复性能阈值断言。 @@ -153,3 +164,14 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 response。配置里的 H100 baseline 因此只是结果 artifact，既不能证明回归阈值，也不能外推到 A3；PR 文本中的“双卡”描述也不能覆盖最终 lane/YAML 的单卡事实。^[PR #5524] ^[PR #5402] ^[PR #5845] + +## OMNI-CI-3b — patched upstream benchmark 必须保持参数与 output subtype 兼容 + +- 触发：upstream `benchmarks/serve.py` 增参，或 Omni 聚合字段只存在于扩展 output subtype。 +- 强制：patched copy 逐版本同步公开参数及控制流；background probe 从独立的最小请求生成，主 workload + 完成后显式停止。Omni-only duplex metrics 用 tolerant attribute reads 聚合，plain-vLLM output 缺字段 + 等价于空集合，不能抹掉已经成功的主结果。 +- 禁止：以“看似无关”删除 upstream 新参数；把 probe 混入 throughput/latency 主样本；直接读取扩展字段 + 导致普通 backend 的结果整体 fallback 成 completed=0。 +- 验收：签名/控制流与 pinned upstream 对照；probe rate 0 与正值覆盖启动/停止；plain 与 Omni output + 混合列表都保留完成数，只有存在时才写 duplex metrics。^[PR #5976] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ce9d31467..d1aa585f4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 584d78c6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2ce15ab7` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index bdd13b91e..f3d4e1d6b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 584d78c6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 2ce15ab7` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -30,6 +30,7 @@ sources: [] |---|---| | 理解共享职责和数据流 | [architecture](architecture.md) | | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | +| vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md new file mode 100644 index 000000000..5c7e7de57 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md @@ -0,0 +1,33 @@ +--- +title: "Diffusion upstream 兼容规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py] +confidence: high +--- + +# Diffusion upstream 兼容规则 + +## DIFF-4k — upstream API shim 与 accuracy backend 必须按能力而非名字对齐 + +- 触发：vLLM/torch bump 删除或重命名 MoE/quant/platform API，或 kernel hub 没有当前 image build。 +- 强制：机械 shim 迁入 Omni 单一共享 owner（如 `copy_missing_attrs`、`is_interleaved`），全部 caller + 同步；MoE 使用 live factory 名，XPU op priority 使用 live provider 名。accuracy comparison 两侧 + 必须走同一 attention capability chain；hub backend 不可用时 server/reference 一起降到 SDPA/native。 +- 禁止：从已删除 upstream 私有 helper 导入；只让一侧 fallback；用 compile-vs-eager LPIPS 判断 + quantization 质量，或因 fallback 放宽所有 backend 的 gate。 +- 验收：import/调用者 census 无旧 symbol；shared helper 保留 tensor attrs；backend probe 对可用/不可用 + 分支断言两侧匹配，阈值只按实际 backend 分支选择；quant quality 两臂都 eager。真实硬件 accuracy + 仍是必要证据，单测不能证明质量门通过。^[PR #5976] + +## DIFF-4l — regional compile wrapper 必须保留被检查 callable 的签名 + +- 触发：包装 regional-compiled `forward`，或升级会用 reflection 发现 DiT block 的 Cache-DiT。 +- 强制：保留参数名和 return annotation；编译 callable 必须仍能被 `inspect.signature` 识别为原 + `forward` 合同。兼容修复若只剩签名透明性，compile failure 必须继续显式暴露。 +- 禁止：用无签名的通用 `*args, **kwargs` wrapper；把已撤回的常驻 eager fallback 写成 active 行为； + 以进程能启动替代 Cache-DiT block discovery 验证。 +- 验收：打开/关闭 Cache-DiT 都能匹配目标 block；测试冻结参数名与 return annotation，并让真实 lazy + compile error 失败而非静默降级。^[PR #5976] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index bdefaafec..224db9dc0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 584d78c6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2ce15ab7` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 9e53b4b90..6158c507f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py] +sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py] confidence: high --- @@ -48,3 +48,13 @@ confidence: high receiver/非 sender+无 hook→创建，并覆盖安全 shutdown。Bagel/Hunyuan KV-only 和 Qwen3-Omni/MiniCPM-o payload 路径各做真实拓扑 smoke。该矩阵不证明 issue #5595 中独立的 shutdown/orphan 问题已解决。 ^[PR #5744] + +## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 + +- 触发：上游 MoE factory/oracle 或 `init_model_parallel_group` 增加 all-to-all manager 要求。 +- 强制：EP group 在运行版本支持 `use_all2all` 时显式启用；兼容旧版本用签名能力检测，不能按版本 + 字符串猜测。非 EP group 保持原参数，避免扩大 communicator 行为。 +- 禁止：创建普通 device communicator 后等 weight load 才触发 manager assert；向不支持该 kwarg + 的旧 vLLM 无条件传参；把 TP/EP 配置存在当作 manager 已初始化。 +- 验收：mock 新旧签名分别断言 EP 转发/旧版省略，普通 group 不启用；目标环境用真实 MoE DiT + TP+EP smoke 验证 manager 非空。^[PR #5976] diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f26c31eaf..69949a021 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 584d78c6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2ce15ab7` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index bcfa81737..3db29fc11 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5976", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -201,6 +201,29 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 记录峰值显存和 dtype。Krea 2 的具体约束见 [Krea 2 规则](../../models/krea2/rules.md)。 ^[PR #4730] +### EXEC-1e — upstream registry 重名时 Omni override 与 plain-vLLM forward 必须同时成立 + +- 触发：上游开始注册 Omni 同名 architecture，或 model runner/forward/capture/dummy-run 接口变化。 +- 强制：全局 registry 无条件重注册 Omni owner；非 staged 模式为 Qwen Omni 选择 thinker，并返回 + stock runner 可消费的 bare tensor；只有 staged talker consumer 存在时才 capture hidden layers。 + runner override 必须接受上游新增 kwargs；text/MoE dummy input 可按 upstream 要求 randomize，但 + code2wav 等结构化 codec id 只能接受参数而不得随机化。worker profiling 同步保存上游新增结果字段。 +- 禁止：因 upstream 已有同名 arch 就跳过 Omni 注册；plain serve 返回 staged tuple；把 vocab-uniform + id 喂给 codec codebook；用 `**kwargs` 隐藏未审查的签名漂移。 +- 验收：registry collision 解析到 Omni class；Qwen2.5/Qwen3 plain thinker 与 staged capture 分别断言 + output shape，PP intermediate 原样透传；dummy-run 覆盖 text randomize 与 generation no-randomize， + 并对 profiling result 做字段 parity。^[PR #5976] + +### EXEC-1f — multimodal wrapper 与 hash algorithm 必须通过 live context 传播 + +- 触发：upstream multimodal input 开始保留原始 bytes、修改 hash 签名或 processor item wrapper。 +- 强制：所有 `get_mm_hashes` / direct `MultiModalHasher.hash_kwargs` 调用传当前 multimodal config 的 + algorithm；processor 通过公开 `get(index)` 解包 `MediaWithBytes`，再把 frames/metadata 交给 HF。 +- 禁止：直接遍历 `.data` 绕过 unwrap；让 stage/replica UUID 前缀替代 content hash algorithm；只修 + 一个模型而不 census 自定义 processor 与 direct hasher caller。 +- 验收：不同 algorithm 进入 hash consumer；wrapped/unwrapped video 都生成同一 HF input，metadata + 保留；stage+replica scope 只包裹 base UUID 且用户 UUID 优先。^[PR #5976] + ### EXEC-2b — fused shard 必须按 source 完整性与布局数值闭环 > H3 text encoder 当前有 fused QKV 与 gate/up owner；Qwen3 code predictor 仍使用分离 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bae707fd9..87cc23fd5 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 584d78c6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2ce15ab7` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index a18c33d13..f8e156838 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: rule tags: [vllm-omni, components, scheduler] -sources: ["vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461"] +sources: ["PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461"] --- # Scheduler 规则 @@ -223,6 +223,17 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 验收：覆盖完整 pair、不同进度、missing/split、parent abort 和 companion abort， 断言请求不会挂死、错误归属保持 request-local、队列和 connector state 都释放。 +## SCHED-6a — async discard 的计数单位必须与 stale drain 一致 + +- 触发：上游 scheduler 改动异步占位、stale output 或 streaming segment replacement。 +- 强制：每个 discard site 用 `num_in_flight_tokens`（scheduled-token 单位）累加 + `num_stale_output_tokens`；每个迟到 frame 先按本次 `num_tokens_scheduled` 结算 in-flight 与 stale + 两个同单位 counter，再在 append/emit 前丢弃。placeholder rollback 与 stale drain 分开记账。 +- 禁止：用 `num_output_placeholders` 初始化 stale counter；清零 placeholder 后仍让迟到输出进入 + placeholder/computed-token 的正常结果更新；把 scheduler 修复宣称为音频 WER 修复。 +- 验收：AR 与 generation 路径覆盖多 token frame、prefill 无 placeholder、连续旧/新 segment， + 断言 stale counter 恰好归零且首个新 segment 输出不被吞；真实质量指标必须另行验证。^[PR #5976] + ## 相关 - 机制与边界见 [architecture](architecture.md)；跨 stage 数据面见 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2c245c883..ea93872e0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 584d78c6` 验证存在 +- 源码校验：以上路径均已在 `main @ 2ce15ab7` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -29,5 +29,6 @@ sources: [] |---|---| | 理解入口到 engine 的边界 | [architecture](architecture.md) | | 根据 PR 描述直达 request contract、streaming format、artifact readiness、factory lifecycle 或 metrics 的规则组与第一批源码 | [rules 与代码地图](rules.md) | +| upstream launcher shutdown、renderer warmup 与多 replica compile-cache 隔离 | [upstream 兼容规则](rules-upstream-compat.md) | | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md new file mode 100644 index 000000000..adbf78deb --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md @@ -0,0 +1,23 @@ +--- +title: "Serving upstream 兼容规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #5976", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, tests/engine/test_stage_engine_startup_cache_env.py] +confidence: high +--- + +# Serving upstream 兼容规则 + +## SERV-7a — upstream launcher 生命周期兼容必须保持模式检测语义 + +- 触发：upstream launcher、renderer warmup 或 shutdown 读取的 engine-client 属性变化。 +- 强制：pure-diffusion client 只在 launcher-facing `vllm_config.shutdown_timeout` 上提供最小 adapter， + 其余属性透明转发，内部 `get_vllm_config()` 仍返回 `None`；chat template warmup 调用 upstream 当前 + owner `online_renderer`。多 replica spawn 时 replica 0 复用默认 compile cache，后续 replica 在持锁 + 的 spawn env 中获得 stage/replica 唯一 `VLLM_CACHE_ROOT`，离开 scope 必须恢复环境。 +- 禁止：让 adapter 改变 pure-diffusion detection；调用已删除的 serving-chat warmup；所有 replica + 共写 AOT cache，或永久污染父进程环境。 +- 验收：pure diffusion 正常 shutdown 且 detection 不变；renderer 恰好 warmup；cache-env 测试覆盖 + replica 0、多个 stage/replica、嵌套恢复和异常恢复。^[PR #5976] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 167c7ba6f..760742674 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 584d78c6`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2ce15ab7`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 89015d5c \ - --to 584d78c6 \ + --from 584d78c6 \ + --to 2ce15ab7 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md index 65f6b288f..7114e9e98 100644 --- a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md +++ b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md @@ -4,7 +4,7 @@ created: 2026-08-10 updated: 2026-08-10 type: rule tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/diffusion/models/lingbot_video/image_condition.py, vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/model_extras/lingbot_video.py, vllm_omni/model_extras/registry.py, tests/diffusion/models/lingbot_video/test_request_utils.py, tests/diffusion/models/lingbot_video/test_image_condition.py, tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py, tests/entrypoints/openai_api/test_image_server.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5311", "Issue #5883"] +sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/diffusion/models/lingbot_video/image_condition.py, vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/model_extras/lingbot_video.py, vllm_omni/model_extras/registry.py, tests/diffusion/models/lingbot_video/test_request_utils.py, tests/diffusion/models/lingbot_video/test_image_condition.py, tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py, tests/entrypoints/openai_api/test_image_server.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5311", "PR #5976", "Issue #5883"] --- # LingBot-Video 规则 @@ -48,6 +48,10 @@ sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/d `seconds` 与 `duration` 同时出现拒绝；显式 `num_frames` 胜过 duration。fps/steps 必须正整数， guidance 非负有限数，shift 正有限数，output type 只允许 `pt|np|latent`。尺寸来源互斥： `width+height`、`size=WIDTHxHEIGHT`、或成对 `resolution+ratio` 三选一，结果必须是 16 的倍数。 + 解析先对每一种尺寸字段做 source merge：online `/v1/videos` 的 `sampling.extra_args` 优先于 + prompt mapping，offline prompt mapping 仅作 fallback；合并后再按表示优先级选择 + `width/height` > `size` > `resolution+ratio`。两条入口必须复用同一个 resolver，不能把某来源 + 整体覆盖另一来源而丢掉未冲突字段。 - 禁止：在 serving 再造 dimension resolver/model-specific width/height alias；把 sampling `num_frames=1` 当成可靠的显式单帧 video——它当前是 image API 的“省略”哨兵，因此会落到默认 81 帧，而 model prompt/extra 的 1 仍是显式值。 @@ -56,7 +60,9 @@ sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/d `(832,480)`，经 resolver 返回标准 `(width,height)=(480,832)`，实际是 portrait；`9:16` 同理 反向。现有测试冻结了该值，不能证明 ratio 名与标准 width:height 语义一致；修复需覆盖全 preset、direct resolver、离线 builder 和两个在线 endpoint 的最终尺寸。sampling sentinel 的 - 不对称也必须先统一 omitted 表示再修改。 + 不对称也必须先统一 omitted 表示再修改。另须用同字段冲突值断言 extra_args 胜过 prompt，并用 + 跨表示冲突冻结 merge 后的 `width/height` > `size` > `resolution+ratio`，同时保留只有 prompt 的 + offline control。^[PR #5976] ## LBV-2a — TI2V 的 VLM 与 VAE 必须共享一次几何对齐及同一 RNG 顺序 diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index 4bb87699b..cf1ba77a9 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: index tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", "PR #5671", vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, docs/design/qwen3_omni_tts_performance_optimization.md] +sources: ["PR #5073", "PR #5671", "PR #5976", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, docs/design/qwen3_omni_tts_performance_optimization.md] --- # Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS） @@ -24,6 +24,10 @@ sources: ["PR #5073", "PR #5671", vllm_omni/model_executor/models/registry.py, v - Qwen3-Omni 的 MUSA ModelOpt FP8 边界是 Thinker 保留量化、Talker/Code2Wav 通过 deploy overlay 显式清除 root quantization metadata，并仅关闭不安全的 Talker-MTP FULL graph wrapper；精确三态与验证边界归下方 architecture 入口。 +- vLLM 0.27 后 upstream 与 Omni registry 有同名 architecture：Omni 必须覆盖全局 entry；plain + `vllm serve` 缺 `model_stage` 时 Qwen2.5/Qwen3 默认 thinker，Qwen3 thinker 在非 staged 模式返回 + bare tensor，只有 staged talker consumer 才请求 captured layers；共享验收见 + [EXEC-1e](../../components/model-executor/rules.md#exec-1e-upstream-registry-重名时-omni-override-与-plain-vllm-forward-必须同时成立)。^[PR #5976] - 官方历史文档：`docs/design/module/archive/async_omni_architecture.md`（以 Qwen3-Omni 为 worked example 的分层运行时快照，非 active spec）、 `docs/design/qwen3_omni_tts_performance_optimization.md`（性能优化实录） diff --git a/knowledge/repos/vllm-omni/tooling/rules.md b/knowledge/repos/vllm-omni/tooling/rules.md index eea20d17b..5c2131d86 100644 --- a/knowledge/repos/vllm-omni/tooling/rules.md +++ b/knowledge/repos/vllm-omni/tooling/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni] -sources: ["PR #5756", "PR #6031", apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py, examples/offline_inference/text_to_image/text_to_image.py, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/examples/offline_inference/test_text_to_image.py] +sources: ["PR #5756", "PR #5976", "PR #6031", apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py, examples/offline_inference/text_to_image/text_to_image.py, vllm_omni/patch.py, tests/diffusion/test_inductor_divisibility_patch.py, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/examples/offline_inference/test_text_to_image.py] confidence: high --- @@ -62,3 +62,14 @@ confidence: high `AttributeError` 变为 320×192 RGB PNG；它是有界的 LingBot E2E artifact 证据，不证明共享 utility、在线 serving 或其他 producer。目标提交没有新增 helper 自动化测试，README full-model case 也不能替代上述类型矩阵。^[PR #6031] + +## OMNI-TOOL-3a — 全局 runtime patch 必须是单调、可自熄的窄证明 + +- 触发：上游 torch/vLLM 升级后需要 monkey-patch 编译器、allocator 或其他进程级对象。 +- 强制：先用 canonical probe 判断上游是否已具备能力；wrapper 只能增加经严格数学/类型条件证明 + 安全的结果，其余全部委托原实现，并带幂等标记。当前 inductor divisibility 补丁只接受不超过 + 20 个符号的 polynomial，且 `cancel(numerator/denominator)` 必须得到整数系数 polynomial。 +- 禁止：吞掉 lazy compile error 后长期静默 eager fallback；对 FloorDiv/ModularIndexing 或非 + polynomial 猜测；补丁安装失败时改变原方法。 +- 验收：覆盖原方法 True、可精确因式消除、非整系数/零分母/非 polynomial/符号上限、重复安装， + 以及“上游已能证明”时不替换方法。真实 FLUX FP8 smoke 另行验证图编译。^[PR #5976] From ee5eea3c4f008856a4e744765c4bf6bd1e19726e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 02:52:06 +0800 Subject: [PATCH 106/465] knowledge(vllm-omni): sync 4697bd1b (PR #5857) Upstream-Commit: 4697bd1be0606b4d4f1fd52b5b8394870016b5ad Upstream-PR: #5857 Knowledge-Delta: bound MiniMax-H3 RTX PRO 5000 topology and performance evidence --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minimax-h3/_index.md | 4 +++- .../models/minimax-h3/rules-deployment.md | 23 ++++++++++++++++++- 11 files changed, 37 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index dccc93d11..ec93634ac 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 584d78c673fe7836809c2a197c3f4843e2fd9ac4 + previous_audited_sha: 2ce15ab7a023d4fb800629e36c06bbe75a4324e2 audited_ref: main - audited_sha: 2ce15ab7a023d4fb800629e36c06bbe75a4324e2 + audited_sha: 4697bd1be0606b4d4f1fd52b5b8394870016b5ad # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 89a5b3392..4e20b1d9c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2ce15ab7a023d4fb800629e36c06bbe75a4324e2`. The canonical machine baseline +4697bd1be0606b4d4f1fd52b5b8394870016b5ad`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d1aa585f4..f91a25dcf 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2ce15ab7` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4697bd1b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f3d4e1d6b..9f9318998 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2ce15ab7` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4697bd1b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 224db9dc0..b07545931 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2ce15ab7` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4697bd1b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 69949a021..381f17c3d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2ce15ab7` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4697bd1b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 87cc23fd5..dd0736804 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2ce15ab7` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4697bd1b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ea93872e0..b9e687207 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2ce15ab7` 验证存在 +- 源码校验：以上路径均已在 `main @ 4697bd1b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 760742674..671379f90 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2ce15ab7`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4697bd1b`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 584d78c6 \ - --to 2ce15ab7 \ + --from 2ce15ab7 \ + --to 4697bd1b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 75ed4841d..757dbf01b 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] confidence: high --- @@ -82,6 +82,8 @@ confidence: high MMH3-3e。 - RTX PRO 6000 的 2/4/8-GPU BF16 profile 保持 TP2 并扩大 Ulysses；单次 T2VA scaling、SM120 backend、PCIe/NUMA 与未测 Ref2VA/替代拓扑边界见 MMH3-3f。 +- RTX PRO 5000 的 2 卡低显存路径使用 20 resident-layer rank-local DLO，4/8 卡使用 resident + TP×Ulysses；topology-aware device order、单请求测量范围及明确未测 Ref2VA 的边界见 MMH3-3h。 ## ComfyUI 请求路由 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 17d347518..0a69fb884 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5863", "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] confidence: high --- @@ -189,3 +189,24 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule `server_test/{fl2va,ref2va}` scripts 不在 commit，raw profiler/benchmark artifact 也未提交；其 output video 只覆盖 Ref2VA Laser case，不是数值 quality gate。后续仍需 same-seed E2E comparison 和重抓 snapshot 证明 churn/reserved 实际下降。^[PR #5891] + +## MMH3-3h — RTX PRO 5000 recipe 的 topology 与证据必须逐 workload 绑定 + +- 触发：引用 2/4/8×72 GiB RTX PRO 5000 的 H3 capacity、latency、device order 或 DLO/resident 推荐。 +- 强制：2 卡只绑定 TP1×Ulysses2、rank-local no-AllGather DLO、20 resident layers、TE-TP2、VAE + patch2、CUDNN/eager；4 卡绑定 TP2×Ulysses2/TE-TP4/VAE4，8 卡 production 推荐绑定 + TP4×Ulysses2/TE-TP8/VAE8，后二者 resident 且不加 eager。物理 GPU ID 只是已测双 NUMA/PCIe host + 的映射示例，复现必须先按 `nvidia-smi topo -m` 重建同等 group locality。 +- 禁止：把 4/8 卡 latency winner 表中其他 topology 写成最终 recipe 推荐；从 20 resident layers 推导 + host master 减少；把 FL2VA first-frame 或相同 encoder/decoder/DiT shape 外推成 Ref2VA 已测。 +- 验收：只把最终 recipe 表内 T2VA 与 first-frame FL2VA 的单 host、单请求、固定 shape/frames/steps + observation 作为有界证据：PyTorch 2.11+cu130、CUDA 13、driver 580.95.05、1344×768、124 frames、 + 50 requested→49 denoise updates。2/4/8 卡推荐路线的 T2VA E2E/denoise/per-update/external peak 分别为 + 515.57/504.69 s/10,300 ms/36.38 GiB、284.01/278.73 s/5,688 ms/67.58 GiB、 + 163.20/159.40 s/3,253 ms/45.83 GiB；first-frame FL2VA 分别为 + 553.45/541.90 s/11,059 ms/36.38 GiB、305.72/299.85 s/6,119 ms/67.58 GiB、 + 171.62/167.25 s/3,413 ms/45.83 GiB。final recipe 写两次 warmup，但 PR body 只明确绑定 4/8 卡， + 因而 2 卡 warmup 范围不能视为已澄清；没有 repeat/variance、immutable image/source/checkpoint pin + 或可复核 quality artifact，且 PR 只有 markdown/pre-commit。review 明确 Ref2VA 未测试，因此 + `MODEL=Ref2VA` 重启说明只是配置建议。重测仍须记录 exact image/SHA、rank groups、NUMA policy、 + all-rank peak 与 artifact/quality。^[PR #5857] From e1ea9437443ca4d04604018983a24d51e76cc29a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 02:59:04 +0800 Subject: [PATCH 107/465] knowledge(vllm-omni): sync 1ff27534 (PR #6045) Upstream-Commit: 1ff275346923cb5c9339bd38e0a594b9aaaa427b Upstream-PR: #6045 Knowledge-Delta: align feature taxonomy, navigation ownership, and support-evidence boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/step-and-batching.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/docs/design-doc-map.md | 7 +++++-- .../docs/generation-and-support-rules.md | 16 +++++++++++++++- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 12 files changed, 33 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ec93634ac..0f37f0359 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2ce15ab7a023d4fb800629e36c06bbe75a4324e2 + previous_audited_sha: 4697bd1be0606b4d4f1fd52b5b8394870016b5ad audited_ref: main - audited_sha: 4697bd1be0606b4d4f1fd52b5b8394870016b5ad + audited_sha: 1ff275346923cb5c9339bd38e0a594b9aaaa427b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4e20b1d9c..0e6791ca6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4697bd1be0606b4d4f1fd52b5b8394870016b5ad`. The canonical machine baseline +1ff275346923cb5c9339bd38e0a594b9aaaa427b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f91a25dcf..2d0dbf32d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4697bd1b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1ff27534` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9f9318998..69bf43234 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4697bd1b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1ff27534` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md index 5111559eb..381516d02 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md +++ b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, components, diffusion] -sources: ["PR #5599", docs/design/feature/diffusion_continuous_batching.md, docs/user_guide/diffusion/execution_modes.md, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/sched/step_scheduler.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_step_pipeline.py] +sources: ["PR #5599", "PR #6045", docs/design/feature/diffusion_continuous_batching.md, docs/user_guide/diffusion/execution_modes.md, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/sched/step_scheduler.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_step_pipeline.py] --- # Step 执行合同与 batching 模式 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b07545931..bf5f945f3 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4697bd1b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1ff27534` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 381f17c3d..7c956e8b6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4697bd1b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1ff27534` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index dd0736804..0a031db84 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4697bd1b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1ff27534` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b9e687207..708c084c2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4697bd1b` 验证存在 +- 源码校验：以上路径均已在 `main @ 1ff27534` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index 1bb74a086..bb71bfff7 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -4,12 +4,12 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, docs] -sources: ["PR #5139", "PR #5833", "PR #5839", "PR #5914", "PR #5958", .github/CODEOWNERS, docs/.nav.yml, docs/design/index.md, docs/design/module/, docs/design/module/archive/README.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] +sources: ["PR #5139", "PR #5833", "PR #5839", "PR #5914", "PR #5958", "PR #6045", docs/features/README.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, .github/CODEOWNERS, docs/.nav.yml, docs/design/index.md, docs/design/module/, docs/design/module/archive/README.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] --- # 官方设计文档地图（docs/design/**） -`main @ 9159cedb` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 按 architecture、feature、 +`main @ 1ff27534` 复核。官方 `docs/design/index.md` 与 `docs/.nav.yml` 按 architecture、feature、 infrastructure/performance 和 21 个 module 页面编排。module frontmatter 的 owner、代码路径、依赖与 验证路径可作 review 首跳 source map；但本批页面全部是 `status: draft`，其中 invariant 编号也明确 只是 candidate，不能覆盖当前代码与测试。尤其 `module/vllm_omni_config.md` 是 @@ -41,6 +41,9 @@ Cosmos3 reasoner/generator 同样在一个 diffusion stage。overview 的 typed PR 只报告 strict MkDocs build（仍有既存 navigation/link warnings）与 diff hygiene，没有 runtime 测试；代表模型的 batching/attention/parallelism/quantization 表须回各 live owner 验证。^[PR #5914] +Feature user guide、design contract、recipe 与 support claim 的分层和 nav 验收见 +[DOCGEN-1c](generation-and-support-rules.md#docgen-1c-feature-taxonomy-只拥有可发现性不生成-support-事实)。^[PR #6045] + | 官方文档 | 一句话 | 知识树 owner | |---|---|---| | `architecture_overview.md` | stage 定义、Qwen3-Omni/HunyuanImage-3.0/MiniMax-H3/Cosmos3 代表映射、指标词汇、system/config/interface 概念图；具体 support 仍回 live owner | [serving](../components/serving/architecture.md)、[configuration](../components/configuration/architecture.md)、[distributed](../components/distributed/architecture.md) + [Qwen](../models/qwen-omni/_index.md)/[Hunyuan](../models/hunyuan-image3/_index.md)/[MiniMax](../models/minimax-h3/_index.md)/[Cosmos3](../models/cosmos3/_index.md) | diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md index 2c9d50dbf..6f34bcba5 100644 --- a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -4,7 +4,7 @@ created: 2026-08-10 updated: 2026-08-10 type: rule tags: [vllm-omni, docs] -sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998"] +sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998", "PR #6045", docs/features/README.md, docs/design/index.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, docs/user_guide/quantization/bitsandbytes.md, docs/user_guide/quantization/gguf.md] --- # Generated docs and supported-model evidence rules @@ -73,3 +73,17 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, 的显式硬件证据；未 linked rows 的硬件列不随本轮 audit 改动。#5969 审计的变化范围仅是 Qwen3-Omni、Qwen-Image、Qwen-Image-2512、Qwen3-TTS CustomVoice、Higgs Audio V3 TTS 的 NVIDIA，以及 MiniMax H3 的 NVIDIA+AMD；这不是其他 row 的平台支持 census。^[PR #5969] + +## DOCGEN-1c — Feature taxonomy 只拥有可发现性，不生成 support 事实 + +- 触发：修改 Features nav/index、design taxonomy、compatibility matrix，或把内容迁到新 guide。 +- 强制：User Guide 按 Runtime and Stage Execution、cross-model Quantization、Diffusion Acceleration、 + Experimental 分组，Custom Pipeline/ComfyUI 归 Integrations；design 定义内部合同，user guide 定义 + 启用/兼容入口，recipe 绑定模型/硬件。Communication 留在 Developer Guide；未稳定的 Pipeline + Parallelism 可 direct-link 但不进入主导航。迁移必须让新页入 nav，并保留旧 heading anchor。 +- 禁止：从任一文档层或 nav presence 单独推出 production support；把 quantization 缩成 diffusion-only； + 把已迁到 OOT plugin 的 GGUF 重新包装成 core-owned 主导航 feature，或因移除主入口而删除仍由 + quantization overview 直达的保留 guide；漏掉仍实现/测试且无其他 discoverability 的 BitsAndBytes。 +- 验收：strict MkDocs + link/nav census；兼容表逐项回 live owner。HunyuanImage3/Helios step 边界分别 + 是 grouped 仅 TORCH_SDPA、仅 `max_num_seqs=1`；HSDP 与 Ulysses/CFG 可组合但与 TP 不兼容；local + layerwise offload 与 multi-device DLO 分表。docs-only build 不证明 runtime。^[PR #6045] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 671379f90..597dd27e4 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4697bd1b`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1ff27534`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 2ce15ab7 \ - --to 4697bd1b \ + --from 4697bd1b \ + --to 1ff27534 \ --repo \ --mode report-only ``` From 4e1afd95a1b8ea266f938646afbece60a3228b83 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 03:17:16 +0800 Subject: [PATCH 108/465] knowledge(vllm-omni): sync 58007c00 (PR #5957) Upstream-Commit: 58007c0084b2a4411da57811e74290365ac80f04 Upstream-PR: #5957 Knowledge-Delta: IndexTTS 2.5, request-end payload, native TTS speed, and XPU compatibility --- adapters/vllm_omni/manifest.yaml | 2 +- adapters/vllm_omni/release_baseline.yaml | 16 ++--- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/benchmark/_index.md | 6 +- .../benchmark/guides/benchmark-scope.md | 13 +++- knowledge/repos/vllm-omni/ci/rules.md | 15 +++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 12 +++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 11 ++- .../vllm-omni/components/serving/_index.md | 2 +- .../serving/rules-upstream-compat.md | 14 +++- .../vllm-omni/components/serving/rules.md | 12 +++- knowledge/repos/vllm-omni/models/catalog.md | 26 +++---- .../vllm-omni/models/indextts-2-5/_index.md | 28 ++++++++ .../models/indextts-2-5/architecture.md | 51 +++++++++++++ .../vllm-omni/models/indextts-2-5/rules.md | 72 +++++++++++++++++++ 20 files changed, 256 insertions(+), 36 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/indextts-2-5/_index.md create mode 100644 knowledge/repos/vllm-omni/models/indextts-2-5/architecture.md create mode 100644 knowledge/repos/vllm-omni/models/indextts-2-5/rules.md diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index cc28a0afe..3469ae77d 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -79,7 +79,7 @@ modules: wave: 1 risk: high input_output: - local_paths: [vllm_omni/inputs/, vllm_omni/outputs/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/serialization.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_engine_core_client.py, vllm_omni/engine/output_processor.py, vllm_omni/request.py] + local_paths: [vllm_omni/inputs/, vllm_omni/outputs/, vllm_omni/utils/mm_outputs.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/serialization.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_engine_core_client.py, vllm_omni/engine/output_processor.py, vllm_omni/request.py] upstream_paths: [vllm/inputs/, vllm/v1/engine/input_processor.py, vllm/v1/engine/output_processor.py, vllm/v1/request.py] test_paths: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/test_async_omni_engine_input.py, tests/test_outputs.py] import_check: "from vllm_omni.inputs.data import *; from vllm_omni.inputs.preprocess import *; from vllm_omni.engine.async_omni_engine import AsyncOmniEngine; from vllm_omni.engine.orchestrator import Orchestrator; from vllm_omni.engine.serialization import serialize_additional_information; print('Input/Output imports OK')" diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0f37f0359..d78cc950b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4697bd1be0606b4d4f1fd52b5b8394870016b5ad + previous_audited_sha: 1ff275346923cb5c9339bd38e0a594b9aaaa427b audited_ref: main - audited_sha: 1ff275346923cb5c9339bd38e0a594b9aaaa427b + audited_sha: 58007c0084b2a4411da57811e74290365ac80f04 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 79 - sha256: 700ab6a4918de2d796604d1e6da5f619731a53bb09b8c56416f253e820ffdf3a + count: 81 + sha256: 6526182d39857c0d9e2e04c1fb4b69606aa8e93b3019f2b471425722a9fcc592 diffusion: count: 60 sha256: ae57a7d6867352714c00b5b93a5f22874ebc2f370cd32067925bb14ec84214f8 pipelines: - count: 52 - sha256: 54a31adef93bc33e2009252699075930ee05ffb38605a52ba9370a21c542b320 + count: 53 + sha256: 152a88ee3e6d7dbb048d77eba3787361ab3820d289a01d562b6a84f4c142a960 deploy_yamls: - count: 80 - sha256: 9fadef14f3ec845ec7a7060df69250badd9a7f002bf5962db229d32c38185397 + count: 81 + sha256: 21e1b8c138a221ede273639d1c05599954489740ebaeff31d0ea3a8b59af3e48 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0e6791ca6..226e0e715 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1ff275346923cb5c9339bd38e0a594b9aaaa427b`. The canonical machine baseline +58007c0084b2a4411da57811e74290365ac80f04`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/benchmark/_index.md b/knowledge/repos/vllm-omni/benchmark/_index.md index bb6522ed0..4281b803b 100644 --- a/knowledge/repos/vllm-omni/benchmark/_index.md +++ b/knowledge/repos/vllm-omni/benchmark/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-07-10 type: index tags: [vllm-omni, benchmark] -sources: [] +sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml] --- # vLLM-Omni 性能与 Profiling @@ -25,3 +25,7 @@ sources: [] | 运行 AR graph 或 Hunyuan benchmark | [benchmark guides](guides/_index.md) | | 调查 profiling、模型加载和性能验证错误 | [benchmark incidents](incidents/_index.md) | | 查询历史结果 | [results](results/_index.md) | + +`benchmarks/tts/model_configs.yaml` 是共享 TTS runner registry；新模型在这里声明 model、deploy、voice/ +reference 与 workload，结果保留解析配置、seed、speed、并发和音频属性。IndexTTS 2.5 条目只证明 +harness 可路由；没有 native 2.5 baseline 时不得声称性能等价。^[PR #5957] diff --git a/knowledge/repos/vllm-omni/benchmark/guides/benchmark-scope.md b/knowledge/repos/vllm-omni/benchmark/guides/benchmark-scope.md index f6a30690d..1512d6c4b 100644 --- a/knowledge/repos/vllm-omni/benchmark/guides/benchmark-scope.md +++ b/knowledge/repos/vllm-omni/benchmark/guides/benchmark-scope.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-07-16 type: guide tags: [vllm-omni, benchmark] -sources: ["#3767", "#3938"] +sources: ["#3767", "#3938", "PR #5957", benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml] --- # vLLM-Omni Benchmark 口径 @@ -166,3 +166,14 @@ candidate PR 只能来自三种来源： - 禁止把第二种结果简称成 “AR 极限”。 **一句话规则**：benchmark PR 是尺子，不是自动成为被测对象；candidate PR 是用户需求，不是 agent 推断。先锁四元组，再启动任何远端长跑。 + +## Shared TTS runner 必须分离 registry identity 和实际 served model + +- `--model` 只选择 `model_configs.yaml` 的 workload/deploy/voice 配置；服务从本地 bundle + 启动时，`--served-model-name` 独立指定发给 `vllm bench serve` 的 model 值，不得 + 因此变更 registry key 或工作负载。 +- 为请求注入 seed 前必须 copy task `extra_body`，不能在多轮 sweep 中污染共享 + registry dict；`--num-warmups` 必须非负；`argparse.REMAINDER` 交给底层 CLI 前只移除 + 首个 literal `--`，其余参数和顺序原样保留。 +- 验收：用 local served name + registry key、连续两轮不同 seed、负 warmup 和带/不带 + separator 的 remainder 构造最终 argv；registry 输入在调用后保持不变。^[PR #5957] diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 3f4d1ef72..071ff8b95 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-02 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5976", docker/Dockerfile.ci, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -175,3 +175,16 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 导致普通 backend 的结果整体 fallback 成 completed=0。 - 验收：签名/控制流与 pinned upstream 对照；probe rate 0 与正值覆盖启动/停止；plain 与 Omni output 混合列表都保留完成数，只有存在时才写 duplex metrics。^[PR #5976] + +## OMNI-CI-3c — XPU release image 从同版 upstream target 构建 + +- 触发：升级 XPU vLLM/torch/Inductor 镜像，或为 compiler 故障改变 model/stage eager gate。 +- 强制：从精确 release tag 的 upstream `Dockerfile.xpu` 构建 `vllm-openai` target 后再叠 Omni；torch、oneCCL、 + triton-xpu 成套 pin。v0.27 使用 triton-xpu 3.7.2并保留 upstream oneCCL。 +- 强制：新 API 未支持时用平台 gate 保留 eager。PyTorch 2.13 XPU Dynamo 的 duplicate-handler + workaround 只覆盖 Qwen2.5 Thinker/Talker 和 Qwen3 shared code predictor，不得扩展成全局 + XPU eager。 +- 禁止：用旧 Omni base target 冒充新版 upstream、卸掉 oneCCL 只补 triton，或以 CUDA + 通过外推 XPU。 +- 验收：核对 image provenance、XPU import/startup，以及这三条路径 compile 未被调用而 + 其他模型保留原能力。^[PR #5957] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2d0dbf32d..6bfd1d034 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1ff27534` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 58007c00` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 69bf43234..6e50549b9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1ff27534` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 58007c00` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index bf5f945f3..ec8a470fa 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1ff27534` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 58007c00` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7c956e8b6..00099a393 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -10,7 +10,7 @@ sources: [] # Model Executor - 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1ff27534` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 58007c00` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 3db29fc11..0c2720f58 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5976", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -246,3 +246,13 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 GQA 错序。平台测试或静态 guard 证明不再引用已删除的 projection 属性。H3 eager text encoder 还须对任何未加载 plain retained parameter 在启动时硬失败；unknown checkpoint key 可继续告警，因为它不会留下 model parameter 未初始化。^[PR #4958] ^[PR #5777] ^[PR #5824] + +### EXEC-1g — request-end payload 延迟 D2H 必须先取得 device snapshot + +- 触发：模型设置 `omni_payload_at_request_end`，或修改 full-payload accumulation、CUDA graph + 和 D2H 策略。 +- 强制：只在显式 opt-in、无 prefix cache 且下游消费完整 payload 时延迟 D2H；每步 + clone device tensor，相容 list 可 pack 后以 views 复原，结束才跨设备，中间 output 为 `None`。 +- 禁止：累计 graph/input-buffer alias，或扩散到普通逐步 payload。 +- 验收：覆盖 source 被下一 step 覆写、ragged list shape/value/alias、普通路径与 + finish/abort 清理。^[PR #5957] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0a031db84..f8439731f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1ff27534` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 58007c00` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index f8e156838..3efe70e62 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461"] --- # Scheduler 规则 @@ -236,5 +236,14 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 ## 相关 +## SCHED-6b — request-end full payload 是显式 admission capability + +- 触发：新增或修改只在 request end 消费完整上游 sequence 的 stage。 +- 强制：coordinator allowlist 与模型 capability 同步；完成前零传输，完成时只 enqueue 一次完整 sequence， + consumer 使用 non-async-chunk topology。IndexTTS 2.5 的精确键是 + `("IndexTTS25S2MelDecoder", "indextts2_5_s2mel_decoder")`。 +- 禁止：按模型名猜测、扩大到整个家族，或逐 token 重复搬运。 +- 验收：allowlisted/non-allowlisted 对照、完成时一次传输、abort 无残留。^[PR #5957] + - 机制与边界见 [architecture](architecture.md)；跨 stage 数据面见 [Distributed 组件](../distributed/_index.md)。 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 708c084c2..d27fa24ac 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1ff27534` 验证存在 +- 源码校验：以上路径均已在 `main @ 58007c00` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md index adbf78deb..c251cec2a 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #5976", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, tests/engine/test_stage_engine_startup_cache_env.py] +sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py] confidence: high --- @@ -21,3 +21,15 @@ confidence: high 共写 AOT cache，或永久污染父进程环境。 - 验收：pure diffusion 正常 shutdown 且 detection 不变；renderer 恰好 warmup；cache-env 测试覆盖 replica 0、多个 stage/replica、嵌套恢复和异常恢复。^[PR #5976] + +## SERV-7b — 加速器镜像落后一版时兼容层必须按能力探测 + +- 触发：CUDA 主线升级 vLLM 后，XPU/NPU 等镜像仍使用前一版 renderer、tool parser + import 或 request layout。 +- 强制：可选 renderer warmup 用属性能力探测；Mistral tool-call parser 只在新 import 不存在时 + 回退旧路径；`OmniRequest` 仅为缺失的 `num_stale_output_tokens` 提供零初值， + 不覆写新版 owner 已有的值。 +- 禁止：用 platform name 猜测 API；为兼容旧版跳过新版必需初始化；将临时 shim 宣称为 + 长期 public API。 +- 验收：新/旧 renderer、两条 Mistral import 和有/无 stale counter 的 request layout 都能初始化， + 且新版既有 counter 原值不变。^[PR #5957] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 2bec726e1..9108e8e08 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] confidence: high --- @@ -371,3 +371,13 @@ confidence: high 请求到 engine 的边界见 [Serving architecture](architecture.md)；公开协议通用检查见 [review contracts](../../../../general/review/guides/reviewer-lens-contracts.md)。 + +### SERV-6d — tokenizer-free 与 native speed 由 adapter capability 闭环 + +- 触发：stage 0 跳过 tokenizer，或 TTS adapter 声明 native speed control。 +- 强制：token-only renderer 仅接受 token IDs/prompt embeddings，chat messages 明确失败。HTTP raw、SSE 与 + non-streaming 由 adapter 校验 native speed，encoder speed 固定 1；generic model 拒绝 `speed != 1`， + WebSocket 在 parity 完成前固定 speed=1。 +- 禁止：模型 duration scaling 后再次 resample，或以 HTTP 支持外推 WebSocket parity。 +- 验收：覆盖 token/chat/embed、native/generic、三种 HTTP 输出与 WebSocket。落后后端只用显式 + capability/version shim 并保留 eager fallback，不能让 CUDA 新参数成为全平台前提。^[PR #5957] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 597dd27e4..67ddec6bb 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1ff27534`（2026-08-11）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 58007c00`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -35,28 +35,28 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| -| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 79 个架构名 / 28 个模型族目录 | +| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 29 个模型族目录 | | Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 60 条 pipeline / 39 个模型族目录 | -| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 52 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 80 份 | +| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 53 个 key | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 81 份 | 对比上一审计快照（`807db6ef`）：AR 架构 72→77，diffusion pipeline 61→58， OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusion 新增 `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `f24a6165` 新增 PersonaPlex 的两个 AR architecture、`personaplex` pipeline key -和 `personaplex.yaml` deploy；diffusion inventory 不变。 +本次相对 `1ff27534` 新增 IndexTTS 2.5 的两个 AR architecture、`indextts2_5` +pipeline key 和 `indextts2_5.yaml` deploy；diffusion inventory 不变。 -## AR/omni 模型族（28） +## AR/omni 模型族（29） aura_omni、audex、bagel、cosyvoice3、covo_audio、dynin_omni、fish_speech、glm_image、 -glm_tts、higgs_audio_v2、higgs_audio_v3、hunyuan_image3、indextts2、 +glm_tts、higgs_audio_v2、higgs_audio_v3、hunyuan_image3、indextts2、indextts2_5、 mammoth_moda2、mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、moss_tts、 moss_tts_nano、omnivoice、personaplex、qwen2_5_omni、qwen3_omni、qwen3_tts、step_audio2、 voxcpm2、voxtral_tts -## Diffusion 模型族（38） +## Diffusion 模型族（39） audiox、bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamid_omni、 dreamzero、ernie_image、flux、flux2、flux2_klein、glm_image、gr00t、helios、 @@ -65,7 +65,7 @@ lingbot_video、longcat_image、longcat_video、ltx2、magi_human、ming_flash_o omnivoice、ovis_image、qwen_image、sd3、sdxl、sensenova_u1、soulx_singer、 stable_audio、wan2_2、z_image、minimax_h3 -## OMNI_PIPELINES key（52） +## OMNI_PIPELINES key（53） Gr00tN1d7（注意:唯一 CamelCase key）、aura_omni、audex_s2s、audex_thinker_only、 audex_tta、audex_tts、bagel、bagel_single_stage、 @@ -78,7 +78,7 @@ ming_tts、ming_tts_moe、minicpmo_4_5、moss_tts_delay、moss_tts_local、 moss_tts_nano、moss_tts_realtime、omnivoice、personaplex、qwen2_5_omni、 qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_tts、 soulxsinger_svc、soulxsinger_svs、step_audio_2、step_audio_2_asr、voxcpm2、 -voxtral_tts、wan2_2_ti2v、nemotron_labs_audex +voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 注意：单 stage diffusion 模型**多数不在** `OMNI_PIPELINES`（引擎为它们生成 默认 diffusion stage 配置,见 [Config 组件](../components/configuration/architecture.md)）; @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex ```bash python tools/audit_vllm_omni_release.py \ - --from 4697bd1b \ - --to 1ff27534 \ + --from 1ff27534 \ + --to 58007c00 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/indextts-2-5/_index.md b/knowledge/repos/vllm-omni/models/indextts-2-5/_index.md new file mode 100644 index 000000000..43dedf9de --- /dev/null +++ b/knowledge/repos/vllm-omni/models/indextts-2-5/_index.md @@ -0,0 +1,28 @@ +--- +title: "IndexTTS 2.5" +created: 2026-09-02 +updated: 2026-09-02 +type: index +tags: [vllm-omni, models] +sources: ["PR #5957", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/indextts2_5.yaml, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/indextts2/, vllm_omni/model_executor/stage_input_processors/indextts2.py, vllm_omni/entrypoints/openai/tts_adapters/indextts2.py, recipes/IndexTeam/IndexTTS-2_5.md] +confidence: high +--- + +# IndexTTS 2.5 + +- registry/pipeline key：`indextts2_5`；architecture 为 + `IndexTTS25TalkerForConditionalGeneration` 与 `IndexTTS25S2MelDecoder`，复用 versioned + IndexTTS2 implementations。 +- 两 stage、同卡、non-streaming pipeline：AR talker 生成完整 semantic codes，S2Mel CFM/DiT + + EnhancedCodec + BigVGAN 输出 22.05 kHz mono audio。标准 deploy 使用 code-only + `use_gpt_latent=false`，Stage 1 必须等 request-end full payload；async chunk 不适用。 +- 支持 reference/uploaded named voice、emotion controls、multilingual frontend 与 native speed；精确 + 拓扑与 versioned conditioning 见 [architecture](architecture.md)，请求、bundle、payload、性能和 + 验证边界见 [rules](rules.md)。 + +## 路由 + +- IndexTTS 2.5 模型、codec、S2Mel、tokenizer、recipe：本 owner。 +- shared TTS HTTP/streaming adapter：[Serving](../../components/serving/_index.md)。 +- request-end payload snapshot 与 runner：[Model Executor](../../components/model-executor/_index.md)。 +- full-payload admission：[Scheduler](../../components/scheduler/_index.md)。 diff --git a/knowledge/repos/vllm-omni/models/indextts-2-5/architecture.md b/knowledge/repos/vllm-omni/models/indextts-2-5/architecture.md new file mode 100644 index 000000000..6db18cc06 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/indextts-2-5/architecture.md @@ -0,0 +1,51 @@ +--- +title: "IndexTTS 2.5 架构" +created: 2026-09-02 +updated: 2026-09-02 +type: architecture +tags: [vllm-omni, models] +sources: ["PR #5957", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/indextts2_5.yaml, vllm_omni/model_executor/models/indextts2/configuration_indextts2.py, vllm_omni/model_executor/models/indextts2/indextts2_talker.py, vllm_omni/model_executor/models/indextts2/indextts2_s2mel_decoder.py, vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py, vllm_omni/model_executor/stage_input_processors/indextts2.py] +confidence: high +--- + +# IndexTTS 2.5 架构 + +## 两 stage 数据流 + +1. `IndexTTS25TalkerForConditionalGeneration` 是 tokenizer-free `LLM_AR` stage。serving 用与 + Talker 相同的 frontend 估算 dummy token span，runner 在该 span 上注入 speaker、language、 + W2V-BERT reference 和 text embeddings。非 RoPE GPT-2 路径使用 Triton attention；标准 + sampling 是 plain vLLM `num_beams=1` 语义。 +2. `IndexTTS25S2MelDecoder` 是 tokenizer-free `LLM_GENERATION` stage。它在 request end 一次 + 接收完整 semantic-code/conditioning payload，经 EnhancedCodec、length regulator、CFM/DiT + 和 BigVGAN 生成 22.05 kHz mono waveform。connector 因此是 `codec_streaming=false`， + pipeline 不允许 async chunk。 + +标准 deploy 的两个 stage 都在 device 0，但这只是默认 placement，不是跨设备能力结论。 + +## 2.0 与 2.5 conditioning 分界 + +| 语义 | IndexTTS 2.0 | IndexTTS 2.5 | +|---|---|---| +| prefill prefix | 34 tokens；Conformer/Perceiver | 3 tokens；projected CAMPPlus speaker + 2 zero rows | +| reference semantic | RepCodec | W2V-BERT sequence | +| language | text frontend | text token + language embedding | +| Stage-1 codec | RepCodec 路径 | EnhancedCodec | +| GPT latent | 默认开 | 官方默认关；实验性 opt-in | + +2.5 的 code-only 路径不在每个 decode step 携带重复 GPU code row；完成时从 request-owned +`output_token_ids` 构造 mel codes，full payload 只携带 conditioning metadata。若显式启用 +GPT latent，code 与 latent 必须同时存在且长度精确一致。 + +## Bundle 与 tokenizer ABI + +- 官方 2.5 local bundle 可以没有 HF `config.json`；仅在 model type 可确认为 + `indextts2_5` 且确实缺文件时合成 versioned config。顶层 `vocab_size` 供 vLLM warmup， + nested GPT defaults 再与显式 override 合并。 +- tokenizer 文件是 `multilingual_zh_ja_yue_char_del.tiktoken`；58,836 个 mergeable ranks + 加 1,673 个 special tokens 构成 60,509 大小的 ABI，加载时精确校验。 +- language-token/embedding 行顺序是 checkpoint ABI，不等于官方质量声明。`zhen` 使用字面 + prefix，但 embedding row 归一到 `common`；服务对外文档化的语言是 zh/en/ja/es/ar。 + +请求语义、精确 placeholder、本地 asset fail-closed 和 dynamic batching 门禁见 +[rules](rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md b/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md new file mode 100644 index 000000000..46cfd8a78 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md @@ -0,0 +1,72 @@ +--- +title: "IndexTTS 2.5 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, models] +sources: ["PR #5957", vllm_omni/deploy/indextts2_5.yaml, vllm_omni/model_executor/models/indextts2/, vllm_omni/model_executor/stage_input_processors/indextts2.py, vllm_omni/entrypoints/openai/tts_adapters/indextts2.py, tests/model_executor/models/indextts2/, tests/entrypoints/openai_api/test_tts_adapter.py, tests/entrypoints/openai_api/test_serving_speech.py] +confidence: high +--- + +# IndexTTS 2.5 规则 + +## ITTS25-1a — native bundle/config 与两 stage topology 必须 fail fast + +- 触发：修改 registry、pipeline、deploy、bundle discovery、tokenizer-free init 或 latent policy。 +- 强制：缺 HF `config.json` 的 synthetic config 只允许本地 `indextts2_5` bundle，且 bundle 内 tokenizer、 + codec、S2Mel/vocoder assets 完整；Stage 0 使用 token-only renderer/TRITON attention，Stage 1 消费完整 + request-end payload。默认 code-only，不得静默启用无官方 reference 的 GPT-latent variant。 +- 禁止：把 model-scoped missing-config fallback 扩展给其他 loader error；用 public asset 替代缺文件； + 对 mel/latent 长度取 min 静默截断，或把 CFM 拆成 async chunks。 +- 验收：registry/config/deploy 双向一致；缺失/错误 bundle、policy、shape 明确失败；code-only 从 completed + output token IDs 构造 mel codes，latent 模式要求同长非空 tensor；seed/conditioning 保留到 Stage 1。 + placeholder span 必须与 embedding 精确相等，不得 pad/truncate：2.5 为 3 个 conditioning prefix + + 与 Talker 共用 lang/normalization/tokenizer/wrapper 过滤的输出（含 start/stop）+ 1 个 + `start_mel`；2.x 旧路径为 34 个 prefix。该精确合同下禁止 chunked prefill。 + +## ITTS25-1b — frontend、voice、emotion 与 speed 必须只有一份请求语义 + +- 触发：修改 tokenizer/text normalization、language、conditioning cache、voice/emotion 或 `speed`。 +- 强制：官方语言为 zh/en/ja/es/ar，`zhen` 只是混合中英 frontend mode；cache salt 包含 text、voice、 + emotion、lang 与 normalization。HTTP native speed 范围 `[0.5,2.0]`，adapter 写 + `duration_factor=1/speed`，audio encoder 保持 1.0，避免二次 resample；model/S2Mel 用共享 bounds + 做 direct-library final guard。发音标记 `` 与标点清理顺序属于 tokenizer ABI； + 必需的 WeText/日语 frontend 依赖缺失时必须给出可操作的 optional-extra 错误， + 不得静默降级为未规范化文本。只有明确允许的 optional Spanish normalizer 可 raw fallback。 +- 禁止：无 reference/named voice 时合成；把 `zhen` 当独立模型语言或把 recipe 中 `yue` 示例当官方 + 支持；HTTP native speed 后再次编码变速；声称 WebSocket 已有 adapter-aware speed parity。 +- 验收：raw/SSE/non-streaming duration 一致且越界 400 后 engine 健康；WebSocket 仍显式要求 speed=1； + language/normalization 冲突改变 cache key，而 speed 不进入 conditioning salt；三种 emotion owner 与 + uploaded voice 都覆盖。 + +request-end device buffer 的共享生命周期只由 +[EXEC-1g](../../components/model-executor/rules.md#exec-1g-request-end-payload-延迟-d2h-必须先取得-device-snapshot) +定义，本 owner 只声明 IndexTTS 2.5 是显式 request-end consumer。 + +## ITTS25-1c — 性能与质量证据必须保留可复核边界 + +- 触发：引用 latency、VRAM、speed ratio、ASR、MPS/XPU 或 production-ready。 +- 强制：结论只绑定 immutable bundle/source、标准两-stage deploy、固定请求/seed、真实 + device identity（至少 capability/SM/HBM）、warmup/repeats、all-process memory、latency/RTF + 与可复核 audio/quality artifact；设备 label 与 runtime 属性冲突时不得擅自选 SKU。 +- 禁止：把约 59.8 GiB configured footprint说成 weight requirement；从十次无增长推出长期无泄漏； + 把 Whisper 单句匹配当通用多语种/voice quality，或声称有 authoritative native 2.5 baseline。 +- 验收：没有可运行、同输入的 native 2.5 baseline 时明确标记 comparison unavailable；短期 + profiler/ASR smoke 不能替代 native-vs-Omni、多语种质量、长期泄漏或 AMD 运行证据。^[PR #5957] + +## ITTS25-1d — S2Mel batching 保持逐请求长度、seed 与数值策略 + +- 触发：修改 S2Mel CFM grouping、WaveNet ragged path、CFG unpadding、noise seed、precision、 + compile/graph 或 vocoder 交付。 +- 强制：保留每请求 code/ref 长度和 `duration_factor`，按 CFM 总长度稳定排序并在 + `s2mel_cfm_batch_size` 内分组；低于 WaveNet `ragged_min_length` 的 row 必须 singleton。变长 + attention 的 row-major unpad index 按“全部 conditional，再全部 unconditional”CFG 顺序复用， + WaveNet 在每个 logical boundary 重建 right reflection；最后恢复原请求顺序、逐条 crop mel， + 再单独调 BigVGAN。 +- 强制：seeded noise 按请求拥有且不受 batch/reorder 影响；全部 unseeded 时保留全局 + RNG 语义。CFM Euler state 始终 FP32，只有 estimator Linear/Conv1d 路径可 BF16。 +- 禁止：用 batch max 污染短样本、在 reorder 后交换 seed、对已给 seed 退回全局 RNG， + 或同时开 DiT compile 和 CUDA graph；graph 只能用于 full-mask shape。 +- 验收：mixed-length/mixed-seed 的 batched 输出与逐条结果对齐；覆盖 singleton 门、CFG index + 顺序、边界 reflection、原顺序/crop、全-unseeded RNG 和 eager/compile 数值容差。标准 + deploy 使用 25 steps、CFG 0.7、CFM batch 4、DiT compile on/graph off。^[PR #5957] From 66ce7e1159f46290a1c5c66b835d0e30626ac927 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 03:26:12 +0800 Subject: [PATCH 109/465] knowledge(vllm-omni): sync 8b0a6779 (PR #6049) Upstream-Commit: 8b0a6779d4dd485fdcf4b6a180ef75ec7bdfb95f Upstream-PR: #6049 Knowledge-Delta: canonical image task prompt ownership --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 7 +++++-- .../components/model-executor/rules.md | 17 ++++++++++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../docs/generation-and-support-rules.md | 15 +++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 10 +++++----- .../repos/vllm-omni/models/cosmos3/rules.md | 8 +++++--- .../vllm-omni/models/lingbot-video/rules.md | 11 ++++++----- .../review/guides/maintainer-pattern-routing.md | 7 ++++--- 14 files changed, 62 insertions(+), 29 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d78cc950b..d47c1e769 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1ff275346923cb5c9339bd38e0a594b9aaaa427b + previous_audited_sha: 58007c0084b2a4411da57811e74290365ac80f04 audited_ref: main - audited_sha: 58007c0084b2a4411da57811e74290365ac80f04 + audited_sha: 8b0a6779d4dd485fdcf4b6a180ef75ec7bdfb95f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 226e0e715..ecfca2ab7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -58007c0084b2a4411da57811e74290365ac80f04`. The canonical machine baseline +8b0a6779d4dd485fdcf4b6a180ef75ec7bdfb95f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6bfd1d034..ea5eaacf2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 58007c00` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8b0a6779` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6e50549b9..995009f47 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 58007c00` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8b0a6779` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ec8a470fa..e867fe296 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 58007c00` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8b0a6779` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 00099a393..5564d1fbc 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -9,8 +9,11 @@ sources: [] # Model Executor -- 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、`vllm_omni/worker/`（gpu_*_worker、gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理：data.py、preprocess.py）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 58007c00` 验证存在；stage 配置已经迁移到 +- 源码入口：`vllm_omni/model_executor/`（layers、model_loader、models、stage_input_processors）、 + `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 + gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 + `vllm_omni/platforms//` +- 源码校验：以上路径均已在 `main @ 8b0a6779` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 0c2720f58..77df55736 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-02 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46"] --- # Model Executor 规则 @@ -25,6 +25,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | H3 text encoder fused owner；其他模型先确认目标 main 是否已有 fused parameter | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | Talker-MTP FULL graph、平台 capture 能力、显式 opt-out | `mtp-graph`：`EXEC-4c` | `platforms/interface.py::supports_talker_mtp_graph_capture` → platform override → model `talker_mtp_graph_safe` → `OmniGPUModelRunner._init_talker_mtp` | +| `model_extras`、shared T2I/I2V example、canonical prompt envelope | `image-task-envelope`：`EXEC-6a` | `examples/offline_inference/{text_to_image/text_to_image.py,image_to_video/image_to_video.py}` → `model_extras/registry.py::{build_text_to_image_prompt,build_image_to_video_prompt}` → model pipeline validation | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -34,6 +35,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`, `EXEC-2b` | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a` | | `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | +| `image-task-envelope` | shared image task example 或 `model_extras` prompt builder | `EXEC-6a` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | ## 严格配置校验 @@ -256,3 +258,16 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 禁止：累计 graph/input-buffer alias，或扩散到普通逐步 payload。 - 验收：覆盖 source 被下一 step 覆写、ragged list shape/value/alias、普通路径与 finish/abort 清理。^[PR #5957] + +### EXEC-6a — shared image example 先建 canonical envelope，model-extra 只做特化变换 + +- 触发：修改 shared T2I/I2V example、`model_extras` prompt registry 或模型 prompt builder。 +- 强制：task runner 先构造完整 canonical dict：T2I 含 `prompt`、`modalities=["image"]`；I2V + 另含 `modalities=["video"]` 与原样 `multi_modal_data`。只有 `negative_prompt is not None` 才写 key， + 因而显式空字符串必须保留。registry 接收该 dict；无 model-specific builder 时 identity-return， + 有 builder 时只翻译模型 token/template/mm kwargs。公共 online serving handler 不参与此离线 seam。 +- 禁止：pipeline 已负责 validation/normalization 时复制 generic builder；让 registry 从零重建 task + modality；用 truthiness 丢空 negative prompt；为每个模型复制 Python example。 +- 验收：canonical builder 覆盖 omitted/empty/value negative prompt 与 PIL media identity；registry + 覆盖 Bagel、MammothModa2、Ming、VACE custom path及 unknown identity path；Cosmos3/LingBot + pipeline tests继续拥有模型专属 validation。^[PR #6049] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f8439731f..466d4f109 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 58007c00` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8b0a6779` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d27fa24ac..5463d63a3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 58007c00` 验证存在 +- 源码校验：以上路径均已在 `main @ 8b0a6779` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md index 6f34bcba5..31da48ba5 100644 --- a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -1,10 +1,10 @@ --- title: "Generated docs and supported-model evidence rules" created: 2026-08-10 -updated: 2026-08-10 +updated: 2026-09-02 type: rule tags: [vllm-omni, docs] -sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998", "PR #6045", docs/features/README.md, docs/design/index.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, docs/user_guide/quantization/bitsandbytes.md, docs/user_guide/quantization/gguf.md] +sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, examples/online_serving/text_to_image/, examples/online_serving/image_to_video/, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998", "PR #6045", "PR #6049", docs/features/README.md, docs/design/index.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, docs/user_guide/quantization/bitsandbytes.md, docs/user_guide/quantization/gguf.md] --- # Generated docs and supported-model evidence rules @@ -87,3 +87,14 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, - 验收：strict MkDocs + link/nav census；兼容表逐项回 live owner。HunyuanImage3/Helios step 边界分别 是 grouped 仅 TORCH_SDPA、仅 `max_num_seqs=1`；HSDP 与 Ulysses/CFG 可组合但与 TP 不兼容；local layerwise offload 与 multi-device DLO 分表。docs-only build 不证明 runtime。^[PR #6045] + +## DOCGEN-1d — shared task example 不得把模型默认伪装成通用合同 + +- 触发：修改 shared T2I/I2V README、Python client、Gradio 或 launch script。 +- 强制：名称与说明按 task 而非某个默认模型表述；通用 client 的 size、steps、CFG、seed 未显式提供时 + 省略，让 server/model default 生效。模型专属 curl recipe 可保留，但必须标清范围。canonical task + envelope 的代码 owner 见 [EXEC-6a](../components/model-executor/rules.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。 +- 禁止：在 shared client 默认发送 Qwen-specific CFG/尺寸/steps，或因脚本默认 model 是 Qwen 就把 UI + 宣称为 Qwen-only；不得把 runner/pipeline synthetic smoke 写成 released-weight quality 证据。 +- 验收：README/CLI/UI/server script 名称一致；optional 字段省略与显式零/空分别测试；shell `bash -n`， + canonical CPU tests，至少一个真实模型 task smoke。^[PR #6049] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 67ddec6bb..e4f34749a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 58007c00`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8b0a6779`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,8 +45,8 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `1ff27534` 新增 IndexTTS 2.5 的两个 AR architecture、`indextts2_5` -pipeline key 和 `indextts2_5.yaml` deploy；diffusion inventory 不变。 +本次相对 `58007c00` 四个 inventory 均无变化；PR #6049 只调整 canonical image-task +prompt 的 example/model-extra ownership，不得从 inventory 不变推断请求合同不变。 ## AR/omni 模型族（29） @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 1ff27534 \ - --to 58007c00 \ + --from 58007c00 \ + --to 8b0a6779 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index 4edc40a8f..e9c2b32b9 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py] confidence: high --- @@ -19,7 +19,7 @@ confidence: high | Cosmos3/Edge、layerwise offload、专有 block | COSMOS-1a | `diffusion/registry.py::_DIFFUSION_MODELS`；`diffusion/models/cosmos3/transformer_cosmos3_edge.py::Cosmos3EdgeVFMTransformer` | | Distilled、SDE scheduler、`t_list` | COSMOS-1b | `diffusion/models/cosmos3/pipeline_cosmos3.py::Cosmos3OmniDiffusersPipeline` 及 scheduler config consumer | | seed/generator、逐步噪声 | COSMOS-2a | `pipeline_cosmos3.py::Cosmos3OmniDiffusersPipeline` 的 request-local sampling 路径 | -| `guidance=0`、request extra | COSMOS-2b | `model_extras/cosmos3.py` → pipeline sampling params | +| `guidance=0`、request extra | COSMOS-2b | `model_extras/cosmos3.py` → pipeline sampling params；canonical task envelope 由 shared example 拥有 | | session manager、UND K/V、`freqs_gen`、CFG branch、request ID | COSMOS-2c | `state_cosmos3_adapter.py` → `pipeline_cosmos3.py` → `transformer_cosmos3.py` | | online/offload/HSDP/VAE parallel 支持声明 | COSMOS-3a | capability 文档/recipe → 对应公开入口与实现路径 | | ROCm、MI350X、AITER、latency/peak memory | COSMOS-3b | recipe 的 measurement commit/protocol → `platforms/rocm/platform.py` backend gate → 目标配置复测 | @@ -57,7 +57,9 @@ confidence: high - 强制：仅以 `is None` 判断未提供；保留显式 `0.0` 并沿 request → sampling params → pipeline 断言。 - 禁止：`value or default`、truthy sentinel。 -- 验收：未提供使用默认，`0.0` 保持零，普通正值保持不变，三类测试都到 consumer。 ^[PR #5001] +- 验收：未提供使用默认，`0.0` 保持零，普通正值保持不变，三类测试都到 consumer。Cosmos3 T2I + builder 已删除：shared example 写 `modalities=["image"]`，registry identity-return，pipeline 保留模型 + validation/normalization；不得把 extra-body registry 误当 task-envelope owner。^[PR #5001] ^[PR #6049] ## COSMOS-2c — session manager 缓存必须按 branch/layer 完整装卸 diff --git a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md index 7114e9e98..f6f7de258 100644 --- a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md +++ b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md @@ -1,10 +1,10 @@ --- title: "LingBot-Video 规则" created: 2026-08-10 -updated: 2026-08-10 +updated: 2026-09-02 type: rule tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/diffusion/models/lingbot_video/image_condition.py, vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/model_extras/lingbot_video.py, vllm_omni/model_extras/registry.py, tests/diffusion/models/lingbot_video/test_request_utils.py, tests/diffusion/models/lingbot_video/test_image_condition.py, tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py, tests/entrypoints/openai_api/test_image_server.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5311", "PR #5976", "Issue #5883"] +sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/diffusion/models/lingbot_video/image_condition.py, vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/model_extras/lingbot_video.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/lingbot_video/test_request_utils.py, tests/diffusion/models/lingbot_video/test_image_condition.py, tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py, tests/entrypoints/openai_api/test_image_server.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5311", "PR #5976", "PR #6049", "Issue #5883"] --- # LingBot-Video 规则 @@ -27,8 +27,8 @@ sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/d - 强制：`modalities=["image"]` 且无输入图是 T2I；`["video"]` 无图是 T2V、恰好一图是 TI2V。 image output 不接收参考图，TI2V 不接收多图；reference video/audio 均拒绝。 modality 缺失且无图仅作 legacy T2V fallback，并 `warning_once`；有图却缺 modality 必须拒绝， - 不能猜 I2V。model-extras 的 T2I/TI2V builder 必须写显式 modality，且 TI2V 只收一个 PIL - image。 + 不能猜 I2V。shared task example 必须写显式 modality；`model_extras` 不再复制 LingBot generic + T2I/TI2V builder，单图 cardinality 与其余 validation 由 pipeline normalization 拥有。 prompt 可是 plain string 或 mapping。mapping 的 `caption` 为标量字符串时保持原文，为结构时 做紧凑 JSON；没有 `caption` 时去除 runtime fields 后把其余结构序列化。空 caption、空结构或 不可序列化值 fail closed。 @@ -62,7 +62,8 @@ sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/d preset、direct resolver、离线 builder 和两个在线 endpoint 的最终尺寸。sampling sentinel 的 不对称也必须先统一 omitted 表示再修改。另须用同字段冲突值断言 extra_args 胜过 prompt，并用 跨表示冲突冻结 merge 后的 `width/height` > `size` > `resolution+ratio`，同时保留只有 prompt 的 - offline control。^[PR #5976] + offline control。Canonical envelope 的共享 ownership 与 identity/custom builder 测试见 + [EXEC-6a](../../components/model-executor/rules.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。^[PR #5976] ^[PR #6049] ## LBV-2a — TI2V 的 VLM 与 VAE 必须共享一次几何对齐及同一 RNG 顺序 diff --git a/knowledge/repos/vllm-omni/review/guides/maintainer-pattern-routing.md b/knowledge/repos/vllm-omni/review/guides/maintainer-pattern-routing.md index 49820ae02..c7f98ae77 100644 --- a/knowledge/repos/vllm-omni/review/guides/maintainer-pattern-routing.md +++ b/knowledge/repos/vllm-omni/review/guides/maintainer-pattern-routing.md @@ -1,10 +1,10 @@ --- title: "PR intent 到 maintainer owner 路由" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-02 type: guide tags: [vllm-omni, review] -sources: ["InferMatrixCopilot Issue #24", "PR #3576", "PR #3642", "PR #4106", "PR #4281", "PR #4341", "PR #4718", "PR #4730", "PR #4980", "PR #5001", "PR #5031", "PR #5037", "PR #5052", "PR #5084", "PR #5087", "PR #5088", "PR #5136", "PR #5157"] +sources: ["InferMatrixCopilot Issue #24", "PR #3576", "PR #3642", "PR #4106", "PR #4281", "PR #4341", "PR #4718", "PR #4730", "PR #4980", "PR #5001", "PR #5031", "PR #5037", "PR #5052", "PR #5084", "PR #5087", "PR #5088", "PR #5136", "PR #5157", "PR #6049"] confidence: high --- @@ -37,7 +37,8 @@ diff 的 producer、consumer、失败路径与链接规则一致。 | PR 描述或风险信号 | 主 owner | 重点 | |---|---|---| | 任意模型名、别名、architecture、pipeline/model registry key | [Model resolver](../../models/catalog.md#direct-模型代码入口) | 搜三处 registry → folder/module/class → AR/diffusion models 目录 | -| HunyuanImage3、`model_extras`、shared task examples、`extra_body`、AR prompt/tokenizer | [HunyuanImage3 rules](../../models/hunyuan-image3/rules.md) | 按描述命中快速代码地图；registry、server/offline seam、默认值与 fallback | +| `model_extras`、shared image-task examples、canonical prompt envelope | [Model Executor rules](../../components/model-executor/rules.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换) | example owns task envelope；registry 只做 model-specific transform；再按命中模型追加 owner | +| HunyuanImage3、`extra_body`、AR prompt/tokenizer | [HunyuanImage3 rules](../../models/hunyuan-image3/rules.md) | registry、server/offline seam、默认值与 fallback | | `tests/diffusion/quantization`、可选包 import、硬件支持文档 | [CI environment](../../ci/guides/ci-environment-gotchas.md) | 未安装环境、真实 kernel、claim 一致性 | | benchmark 脚本、percentile、warmup、replica isolation | [performance evidence](../../benchmark/guides/performance-evidence.md) | 计时、统计、失败退出、完整 key set | | 模块移动、compat shim、重复 class/schema | [API surface](../../../../general/review/guides/code-taste-api-surface.md) | identity、旧行为、返回合同 | From 6838d0f25c50ae8474d058224f6b2f3e8af1bc77 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 03:32:59 +0800 Subject: [PATCH 110/465] knowledge(vllm-omni): sync bdb1d996 (PR #5981) Upstream-Commit: bdb1d996bee26e34f59f0f7fe24f928b4bb21996 Upstream-PR: #5981 Knowledge-Delta: loader-scoped diffusion parameter dtype --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-tensor-dtype.md | 30 +++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 9 +----- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 10 +++---- 11 files changed, 46 insertions(+), 22 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-tensor-dtype.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d47c1e769..ce3697a50 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 58007c0084b2a4411da57811e74290365ac80f04 + previous_audited_sha: 8b0a6779d4dd485fdcf4b6a180ef75ec7bdfb95f audited_ref: main - audited_sha: 8b0a6779d4dd485fdcf4b6a180ef75ec7bdfb95f + audited_sha: bdb1d996bee26e34f59f0f7fe24f928b4bb21996 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ecfca2ab7..7c8ff015c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8b0a6779d4dd485fdcf4b6a180ef75ec7bdfb95f`. The canonical machine baseline +bdb1d996bee26e34f59f0f7fe24f928b4bb21996`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ea5eaacf2..196f68bc2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8b0a6779` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bdb1d996` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 995009f47..4fce44d46 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8b0a6779` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bdb1d996` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -32,6 +32,7 @@ sources: [] | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | +| runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-tensor-dtype.md b/knowledge/repos/vllm-omni/components/diffusion/rules-tensor-dtype.md new file mode 100644 index 000000000..466e29004 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-tensor-dtype.md @@ -0,0 +1,30 @@ +--- +title: "Diffusion tensor dtype 规则" +created: 2026-09-03 +updated: 2026-09-03 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #5067", "PR #5068", "PR #5174", "PR #5981", "Issue #5880", vllm_omni/diffusion/layers/norm.py, tests/diffusion/layers/test_norm.py] +confidence: high +--- + +# Diffusion tensor dtype 规则 + +## DIFF-1c — runtime tensor 派生 consumer dtype，parameter 跟随 loader 构造上下文 + +- 触发：mixed precision pipeline 中新建 runtime mask/index/constant/buffer/precomputed tensor，或构造 + shared layer parameter。 +- 强制：runtime tensor 从输入或目标权重派生 device/dtype，整数 index 显式使用 consumer 类型； + graph/eager 与 batch reorder 后保持 shape、alias 和数值语义。parameter constructor 的 + `dtype=None` 则读取 `torch.get_default_dtype()`，但只依赖 loader 的 + `set_default_torch_dtype(od_config.dtype)` 构造作用域；显式 dtype 始终优先。RMSNorm parameter + storage dtype 与 `forward_native` FP32 accumulation 是两个独立合同。 +- 禁止：runtime temporary 依赖环境中不受控的 process-default fp32/CPU 或隐式 cast/move；但不能把 + 这条禁令错用于 loader-scoped parameter construction。不得把 Python 参数默认写死 fp32 来绕过 + BF16 loader context，也不能只测 shape 而不执行真实 consumer。 +- 验收：runtime 路径覆盖 BF16、错长、batch reorder 和 graph/eager；constructor 覆盖 loader-scoped + BF16、FP32 与显式 override。shared RMSNorm 被 HunyuanImage3、Cosmos3、MiniMax-H3、Wan2.2、 + Z-Image 等消费，修复不能只跑一个模型。 +- 证据边界：相同输入的 online/offline 与 retry 重复得到相同失败值，只支持“确定性数值漂移”； + 未绑定软件 commit、硬件与可复核 artifact 的截图不能建立跨硬件或 bit-exact parity。 + ^[PR #5067] ^[PR #5068] ^[PR #5174] ^[PR #5981] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 52669d76f..acc08583c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -67,14 +67,7 @@ confidence: high 到达 consumer。Cosmos3 的落地约束见 [Cosmos3 规则](../../models/cosmos3/rules.md)。 ^[PR #5001] -### DIFF-1c — 新 tensor 从真实 consumer 派生 dtype 和 device - -- 触发：mixed precision pipeline 中新建 mask、index、constant、buffer 或预计算 tensor。 -- 强制：从输入或目标权重派生 device/dtype，整数 index 显式使用 consumer 要求的类型； - graph/eager 与 batch 重排后保持 shape、alias 和数值语义。 -- 禁止：依赖 process 默认 fp32/CPU 或隐式 cast/move；只测 shape 而不执行 BF16 consumer。 -- 验收：BF16、错长、batch reorder 和 graph/eager 用例断言 consumer 收到相同 device/dtype、 - alias 与结果。 ^[PR #5067] ^[PR #5068] ^[PR #5174] ^[PR #5981] +mixed-precision tensor/parameter dtype 合同见 [DIFF-1c 专页](rules-tensor-dtype.md)。 async output readiness、per-worker result channel、shutdown 与 constructor cleanup 合同见 [DIFF-1d/1g/1h/1j 专页](rules-output-lifecycle.md)。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e867fe296..710e1d18b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8b0a6779` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bdb1d996` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5564d1fbc..99a0a7460 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8b0a6779` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bdb1d996` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 466d4f109..3669724a2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8b0a6779` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bdb1d996` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5463d63a3..bc941abe3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8b0a6779` 验证存在 +- 源码校验：以上路径均已在 `main @ bdb1d996` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e4f34749a..166a9d552 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8b0a6779`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bdb1d996`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,8 +45,8 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `58007c00` 四个 inventory 均无变化；PR #6049 只调整 canonical image-task -prompt 的 example/model-extra ownership，不得从 inventory 不变推断请求合同不变。 +本次相对 `8b0a6779` 四个 inventory 均无变化；PR #5981 修正 shared diffusion RMSNorm +parameter 默认 dtype，不得从 inventory 不变推断数值合同不变。 ## AR/omni 模型族（29） @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 58007c00 \ - --to 8b0a6779 \ + --from 8b0a6779 \ + --to bdb1d996 \ --repo \ --mode report-only ``` From 55ccc5fecbd72367bfb9794fdc0bb5232bfc393c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 03:37:55 +0800 Subject: [PATCH 111/465] knowledge(vllm-omni): sync b792578d (PR #6079) Upstream-Commit: b792578d890e73d0371ac7d0d5eb120b1ab90d63 Upstream-PR: #6079 Knowledge-Delta: document MiniCPM-o ready perf completion-only gates --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 11 +++++++++++ .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 10 +++++----- .../repos/vllm-omni/models/minicpm-o-4-5/rules.md | 6 +++++- 11 files changed, 30 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ce3697a50..f8a66cc03 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8b0a6779d4dd485fdcf4b6a180ef75ec7bdfb95f + previous_audited_sha: bdb1d996bee26e34f59f0f7fe24f928b4bb21996 audited_ref: main - audited_sha: bdb1d996bee26e34f59f0f7fe24f928b4bb21996 + audited_sha: b792578d890e73d0371ac7d0d5eb120b1ab90d63 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7c8ff015c..0cdad0a2b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bdb1d996bee26e34f59f0f7fe24f928b4bb21996`. The canonical machine baseline +b792578d890e73d0371ac7d0d5eb120b1ab90d63`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 071ff8b95..52ae41804 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -163,6 +163,17 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 runner 命令没有阈值开关；simplex 只 gate 完成请求数，duplex 另要求每 session 恰好四个音频 response。配置里的 H100 baseline 因此只是结果 artifact，既不能证明回归阈值，也不能外推到 A3；PR 文本中的“双卡”描述也不能覆盖最终 lane/YAML 的单卡事实。^[PR #5524] + ready perf 复用同一合同：Seed-TTS English、`openai-chat-omni`、text+audio、关闭 thinking 并启用 + TTS template，sweep 是 `(concurrency,num_prompts)=(1,10),(4,40)`。CUDA lane 分配 `h100_1`；NPU + lane 分配 `a3_npu_2`，但 JSON hardware mark 声明每 case `num_cards: 1`，资源 allocation 不等于 + 模型 stage 用卡数。两边都直接调用 runner、没有 `--assert-baseline`，所以仍只 gate + `completed == num_prompts`；H100 metrics bucket 在 A3 run 中也只是 artifact，不能构成 A3 threshold。 + ^[PR #6079] +- ready 的 `source_file_dependencies` 只列 perf JSON、MiniCPM deploy/model/stage processor；shared + `run_benchmark.py`/DFX conftest 由 nightly 覆盖，故不触发这个高成本 model job。functional + offline/online/duplex jobs 不可因 perf job 存在而删除：perf 只发 Seed-TTS text prompt，不覆盖 offline、 + non-stream text content assertion 或 image/video/audio multimodal ingestion。若要降成本，应收窄各自 + workload，而不是声称 perf 是 functional superset。^[PR #6079] ^[PR #5402] ^[PR #5845] ## OMNI-CI-3b — patched upstream benchmark 必须保持参数与 output subtype 兼容 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 196f68bc2..27bdf90f6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bdb1d996` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b792578d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 4fce44d46..4a3a43fd8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bdb1d996` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b792578d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 710e1d18b..2ca8aa4bb 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bdb1d996` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b792578d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 99a0a7460..e344cfe14 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bdb1d996` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b792578d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3669724a2..caf922b5b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bdb1d996` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b792578d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index bc941abe3..9da679d28 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bdb1d996` 验证存在 +- 源码校验：以上路径均已在 `main @ b792578d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 166a9d552..e5a2dff1b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bdb1d996`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b792578d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,8 +45,8 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `8b0a6779` 四个 inventory 均无变化；PR #5981 修正 shared diffusion RMSNorm -parameter 默认 dtype，不得从 inventory 不变推断数值合同不变。 +本次相对 `bdb1d996` 四个 inventory 均无变化；PR #6079 增加 MiniCPM-o 4.5 ready perf +lane/DFX config，不得从 inventory 不变推断 CI gate 不变。 ## AR/omni 模型族（29） @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 8b0a6779 \ - --to bdb1d996 \ + --from bdb1d996 \ + --to b792578d \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 82e977e5d..99fedc17d 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -219,13 +219,17 @@ confidence: high evaluator 分配两卡资源；资源 allocation 不等于模型 stage 用卡数。最终 simplex/duplex perf 是 单卡 H100，baseline 只有 H100 bucket，runner 只检查完成数，duplex 再检查四轮 audio count；没有 性能阈值 consumer，故 baseline、TTFT/TTFP/RTF 只构成测量 artifact，不能证明回归或 A3 性能。 + ready 的 trimmed simplex config 仅保留 concurrency 1/4 与 prompts 10/40；CUDA `h100_1`、NPU + `a3_npu_2` 都运行同一 JSON，但 JSON 的 case cardinality 是一张卡且只含 H100 baseline。该 config + 与 runner 没有 threshold consumer；因此 “ready gate” 只表示请求完成 gate 已接线，不是性能 + regression threshold，也不能单凭配置存在声称 H100/A3 实测通过。 - 禁止：把结果 artifact 中存在 baseline 描述成 active performance gate；把 PR 的“双卡”文字覆盖到 最终单卡 lane/YAML；或从 H100 bucket 外推 A3 的性能与阈值。 - 验收：固定上述 collection、阈值、逐轮 cardinality 与 fallback；分别故障注入缺轮、重复音频、 非单调时间和错误 backend。PR 报告的 Daily-Omni 78.53%、Seed-TTS WER 0.0332/0.0255 只能绑定其 日志环境：数据/模型 revision 未固定，Daily 首次受损坏 decord 失败后重跑，kernel warmup 另有 未提交 patch，证据脚本还吞掉 step failure，且把版本号写作 commit。因而不能从这些数字声称 - merge target 的可复现质量/性能通过。^[PR #5524] + merge target 的可复现质量/性能通过。^[PR #5524] ^[PR #6079] ## MCPMO-5b — online serving CI 必须显式区分 chunk 模式与 duplex fixture 语义 From ae8cb01d9ebcfa1352c0ffb5e36ed4c9873c8e61 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 03:43:46 +0800 Subject: [PATCH 112/465] knowledge(vllm-omni): sync 854bff7f (PR #6062) Upstream-Commit: 854bff7fe57383b0aae49ff4a2a59ca0003bcc03 Upstream-PR: #6062 Knowledge-Delta: define Wan VAE spatial reshard boundary invariants --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-wan-spatial-shard.md | 30 +++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 2 ++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 10 +++---- .../repos/vllm-omni/models/wan2-2/_index.md | 3 ++ 12 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-wan-spatial-shard.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f8a66cc03..86e98a69f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bdb1d996bee26e34f59f0f7fe24f928b4bb21996 + previous_audited_sha: b792578d890e73d0371ac7d0d5eb120b1ab90d63 audited_ref: main - audited_sha: b792578d890e73d0371ac7d0d5eb120b1ab90d63 + audited_sha: 854bff7fe57383b0aae49ff4a2a59ca0003bcc03 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0cdad0a2b..9a32c8a27 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b792578d890e73d0371ac7d0d5eb120b1ab90d63`. The canonical machine baseline +854bff7fe57383b0aae49ff4a2a59ca0003bcc03`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 27bdf90f6..66cb61b29 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b792578d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 854bff7f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 4a3a43fd8..b60f4bdf3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b792578d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 854bff7f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -33,6 +33,7 @@ sources: [] | vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | +| Wan VAE height/width spatial reshard、empty tail、attention extent | [Wan spatial-shard rules](rules-wan-spatial-shard.md) | | multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-wan-spatial-shard.md b/knowledge/repos/vllm-omni/components/diffusion/rules-wan-spatial-shard.md new file mode 100644 index 000000000..6dd977777 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-wan-spatial-shard.md @@ -0,0 +1,30 @@ +--- +title: "Wan VAE spatial-shard 规则" +created: 2026-09-02 +updated: 2026-09-02 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #6062", vllm_omni/diffusion/distributed/autoencoders/wan_spatial_shard.py, tests/diffusion/distributed/test_wan_spatial_shard.py] +confidence: high +--- + +# Wan VAE spatial-shard 规则 + +## DIFF-3b — trimmed global extent 必须可逆地 reshard 为固定 local shape + +- 触发：修改 Wan VAE spatial-shard height/width split、gather/trim/reshard、attention wrapper、padding + 或保存的 `SpatialShardContext`。 +- 强制：reshard 使用 process group 的相对 rank，并在存在 context 时先验证它与 context rank 一致； + 不一致必须在 slicing 前 fail-fast。设 `actual_extent=x.shape[dim]`，则 + `start=min(rank*local_extent, actual_extent)`，valid extent 同时受 context 推导的有效长度和 + `actual_extent-start` 约束。`narrow` 后不足 `local_extent` 的尾部补零，使每 rank shape 相同； + 空 trailing shard 必须成为合法的 zero-length slice + full zero padding，而不是越界 offset。 +- attention wrapper 在 global gather 后把目标 extent clamp 到实际 gathered extent；attention 输出在 + split axis 上必须保持这个 trimmed extent，否则报告 group rank、split axis、local/gathered/trimmed/ + output extent 和 context 后失败，禁止把改变后的 extent 继续按旧边界 reshard。 +- 禁止：仅把 valid length clamp 为零却保留越界 start；让最后一 rank 返回短 tensor；混用 global rank + 与 group-relative rank；假设 attention 永远 shape-preserving 而无 guard；只测 height 后外推 width。 +- 验收：height/width 都覆盖 world size 大于 extent 的 empty tail、不能整除的 partial tail 和常规 extent； + 拼接所有固定 local shards 并 trim 后必须逐值重建输入。另覆盖 context/group rank mismatch 与 attention + shrink/expand。CPU slicing tests 只证明边界数学；collective/halo correctness 仍需多 GPU height/width 对 + single-process reference，生产 pipeline smoke 另验证真实 VAE patch/caller。^[PR #6062] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index acc08583c..688ed9c0c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -30,6 +30,7 @@ confidence: high | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | | modular/multi-DiT、`_dit_modules`、request-scoped Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2j` | request batch key → model policy/cache runtime；pipeline component list → shared consumers | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | +| Wan VAE spatial shard、gather/trim/reshard、empty tail、attention extent | `distributed-vae`：`DIFF-3b` | `distributed/autoencoders/wan_spatial_shard.py` → patched decoder attention/conv → gather final frame | | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4j` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | @@ -39,6 +40,7 @@ confidence: high | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1j` | | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2n` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | +| `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2ca8aa4bb..a1397ac92 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b792578d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 854bff7f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e344cfe14..80a0692d8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b792578d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 854bff7f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index caf922b5b..1aca0466b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b792578d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 854bff7f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9da679d28..66c51f25c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b792578d` 验证存在 +- 源码校验：以上路径均已在 `main @ 854bff7f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e5a2dff1b..1e6189cde 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b792578d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 854bff7f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,8 +45,8 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `bdb1d996` 四个 inventory 均无变化；PR #6079 增加 MiniCPM-o 4.5 ready perf -lane/DFX config，不得从 inventory 不变推断 CI gate 不变。 +本次相对 `b792578d` 四个 inventory 均无变化；PR #6062 修正 Wan VAE distributed spatial +reshard boundary，不得从 inventory 不变推断并行数值合同不变。 ## AR/omni 模型族（29） @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from bdb1d996 \ - --to b792578d \ + --from b792578d \ + --to 854bff7f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/wan2-2/_index.md b/knowledge/repos/vllm-omni/models/wan2-2/_index.md index 7914edbbe..7592b956f 100644 --- a/knowledge/repos/vllm-omni/models/wan2-2/_index.md +++ b/knowledge/repos/vllm-omni/models/wan2-2/_index.md @@ -43,6 +43,9 @@ sources: ["PR #2783", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/mo - 依赖共享模块：[Diffusion 组件](../../components/diffusion/_index.md) （CFG-parallel、SP、PP mixin、分布式 VAE）、`diffusion/models/dmd2/` 蒸馏 mixin。 +- Wan VAE spatial height/width gather→trim→reshard 的 empty/partial-tail、rank context 与 attention + extent 合同由共享 [DIFF-3b](../../components/diffusion/rules-wan-spatial-shard.md#diff-3b-trimmed-global-extent-必须可逆地-reshard-为固定-local-shape) 拥有； + 该 VAE patch 会影响所有复用 Wan autoencoder 的 pipeline，不应只在本家族内修。^[PR #6062] ## 目录内容 From 3267ec6e289b7cf1b1549a26f4ff4382573c6383 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 03:48:59 +0800 Subject: [PATCH 113/465] knowledge(vllm-omni): sync 3c3989ed (PR #6074) Upstream-Commit: 3c3989ed04300d396b3f6b44e9c0b70a0f9cabe0 Upstream-PR: #6074 Knowledge-Delta: align quantization navigation evidence boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/docs/generation-and-support-rules.md | 11 +++++++---- knowledge/repos/vllm-omni/models/catalog.md | 10 +++++----- 10 files changed, 21 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 86e98a69f..bcdaad86b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b792578d890e73d0371ac7d0d5eb120b1ab90d63 + previous_audited_sha: 854bff7fe57383b0aae49ff4a2a59ca0003bcc03 audited_ref: main - audited_sha: 854bff7fe57383b0aae49ff4a2a59ca0003bcc03 + audited_sha: 3c3989ed04300d396b3f6b44e9c0b70a0f9cabe0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9a32c8a27..f9a024b9c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -854bff7fe57383b0aae49ff4a2a59ca0003bcc03`. The canonical machine baseline +3c3989ed04300d396b3f6b44e9c0b70a0f9cabe0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 66cb61b29..11e5b6e9a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 854bff7f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3c3989ed` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b60f4bdf3..a2acf9f73 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 854bff7f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 3c3989ed` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a1397ac92..b1c8b2a0e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 854bff7f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3c3989ed` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 80a0692d8..61a8cde80 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 854bff7f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3c3989ed` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1aca0466b..465434c00 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 854bff7f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3c3989ed` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 66c51f25c..93e3bef8b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 854bff7f` 验证存在 +- 源码校验：以上路径均已在 `main @ 3c3989ed` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md index 31da48ba5..cdb0e5163 100644 --- a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -4,7 +4,7 @@ created: 2026-08-10 updated: 2026-09-02 type: rule tags: [vllm-omni, docs] -sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, examples/online_serving/text_to_image/, examples/online_serving/image_to_video/, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998", "PR #6045", "PR #6049", docs/features/README.md, docs/design/index.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, docs/user_guide/quantization/bitsandbytes.md, docs/user_guide/quantization/gguf.md] +sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, examples/online_serving/text_to_image/, examples/online_serving/image_to_video/, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998", "PR #6045", "PR #6049", "PR #6074", docs/features/README.md, docs/design/index.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, docs/user_guide/quantization/overview.md, docs/user_guide/quantization/bitsandbytes.md, docs/user_guide/quantization/gguf.md] --- # Generated docs and supported-model evidence rules @@ -82,11 +82,14 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, 启用/兼容入口，recipe 绑定模型/硬件。Communication 留在 Developer Guide；未稳定的 Pipeline Parallelism 可 direct-link 但不进入主导航。迁移必须让新页入 nav，并保留旧 heading anchor。 - 禁止：从任一文档层或 nav presence 单独推出 production support；把 quantization 缩成 diffusion-only； - 把已迁到 OOT plugin 的 GGUF 重新包装成 core-owned 主导航 feature，或因移除主入口而删除仍由 - quantization overview 直达的保留 guide；漏掉仍实现/测试且无其他 discoverability 的 BitsAndBytes。 + 把已迁到 OOT plugin 的 GGUF 重新包装成 core-owned 主导航 feature，或因它同时从主导航和 overview + 消失就宣称 runtime/guide/tests 已删除；漏掉已有 core 实现、测试与 guide 的 BitsAndBytes。BitsAndBytes + 的 CUDA method presence 不等于所有 NVIDIA 型号或模型都已验证：当前 diffusion 明确验证范围只有 + Z-Image-Turbo，其他 diffusion、Omni/TTS、多 stage 组合仍按未验证处理。 - 验收：strict MkDocs + link/nav census；兼容表逐项回 live owner。HunyuanImage3/Helios step 边界分别 是 grouped 仅 TORCH_SDPA、仅 `max_num_seqs=1`；HSDP 与 Ulysses/CFG 可组合但与 TP 不兼容；local - layerwise offload 与 multi-device DLO 分表。docs-only build 不证明 runtime。^[PR #6045] + layerwise offload 与 multi-device DLO 分表。docs-only build 不证明 runtime；overview 移除 GGUF 只同步 + core 导航归属，保留的 GGUF guide、OOT plugin 说明与实现/测试证据仍须独立核查。^[PR #6045] ^[PR #6074] ## DOCGEN-1d — shared task example 不得把模型默认伪装成通用合同 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1e6189cde..0a086f73e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 854bff7f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3c3989ed`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,8 +45,8 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b792578d` 四个 inventory 均无变化；PR #6062 修正 Wan VAE distributed spatial -reshard boundary，不得从 inventory 不变推断并行数值合同不变。 +本次相对 `854bff7f` 四个 inventory 均无变化；PR #6074 只对齐 quantization overview 与 +导航中的 BitsAndBytes/GGUF 可发现性，不是 runtime registry 或支持范围变更。 ## AR/omni 模型族（29） @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b792578d \ - --to 854bff7f \ + --from 854bff7f \ + --to 3c3989ed \ --repo \ --mode report-only ``` From 8725a1c194049c82db9ecfde1da69cfa15e3c126 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 03:53:00 +0800 Subject: [PATCH 114/465] knowledge(vllm-omni): sync a15b5b4b (PR #6046) Upstream-Commit: a15b5b4bcda33a9fbd24017381d1f9423d1e840d Upstream-PR: #6046 Knowledge-Delta: define diff-scoped Python examples policy --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/docs/repository-skills.md | 20 +++++++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 10 +++++----- 10 files changed, 32 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bcdaad86b..ce92afd75 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 854bff7fe57383b0aae49ff4a2a59ca0003bcc03 + previous_audited_sha: 3c3989ed04300d396b3f6b44e9c0b70a0f9cabe0 audited_ref: main - audited_sha: 3c3989ed04300d396b3f6b44e9c0b70a0f9cabe0 + audited_sha: a15b5b4bcda33a9fbd24017381d1f9423d1e840d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f9a024b9c..a1671e7ba 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3c3989ed04300d396b3f6b44e9c0b70a0f9cabe0`. The canonical machine baseline +a15b5b4bcda33a9fbd24017381d1f9423d1e840d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 11e5b6e9a..7b919d802 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3c3989ed` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a15b5b4b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a2acf9f73..3e56b0173 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3c3989ed` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ a15b5b4b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b1c8b2a0e..29ce110f0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3c3989ed` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a15b5b4b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 61a8cde80..be6f1d948 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3c3989ed` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a15b5b4b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 465434c00..bea772b13 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3c3989ed` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a15b5b4b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 93e3bef8b..08e75401f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3c3989ed` 验证存在 +- 源码校验：以上路径均已在 `main @ a15b5b4b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/repository-skills.md b/knowledge/repos/vllm-omni/docs/repository-skills.md index 426a40111..77b7f2ebc 100644 --- a/knowledge/repos/vllm-omni/docs/repository-skills.md +++ b/knowledge/repos/vllm-omni/docs/repository-skills.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-02 type: rule tags: [vllm-omni, docs] -sources: ["PR #6029", .claude/skills/readme.md, .claude/skills/add-diffusion-model/SKILL.md, .claude/skills/add-tts-model/SKILL.md, .claude/skills/diffusion-perf-opt/SKILL.md, .claude/skills/precheck-pr/SKILL.md, .claude/skills/quantization/SKILL.md, .claude/skills/review-pr/SKILL.md, .claude/skills/vllm-omni-npu-upgrade/SKILL.md, .claude/skills/vllm-omni-test/SKILL.md, docs/contributing/README.md] +sources: ["PR #6029", "PR #6046", .claude/skills/readme.md, .claude/skills/add-diffusion-model/SKILL.md, .claude/skills/add-tts-model/SKILL.md, .claude/skills/diffusion-perf-opt/SKILL.md, .claude/skills/precheck-pr/SKILL.md, .claude/skills/precheck-pr/references/checklists.md, .claude/skills/precheck-pr/references/examples-policy.md, .claude/skills/quantization/SKILL.md, .claude/skills/review-pr/SKILL.md, .claude/skills/vllm-omni-npu-upgrade/SKILL.md, .claude/skills/vllm-omni-test/SKILL.md, docs/contributing/README.md] confidence: high --- @@ -19,7 +19,7 @@ confidence: high - 触发：增删/重命名 `.claude/skills/*`、修改 frontmatter `name`，或更新 contributor task-to-skill table 与 `.claude/skills/readme.md`。 - 强制：以目标 revision 的目录与每份 `SKILL.md` frontmatter 为准，分别校验链接路径和 invocation - name。`main @ c4a14091` 恰有 8 个目录且都有 `SKILL.md`：`add-diffusion-model`、 + name。`main @ a15b5b4b` 恰有 8 个目录且都有 `SKILL.md`：`add-diffusion-model`、 `add-tts-model`、`diffusion-perf-opt`、`precheck-pr`、`quantization`、`review-pr`、 `vllm-omni-npu-upgrade`、`vllm-omni-test`。NPU 的目录/链接名是 `vllm-omni-npu-upgrade`，frontmatter 与展示的调用名是 @@ -51,3 +51,19 @@ confidence: high - 验收：交付记录 task authority、选用的 skill 与理由、实际命令/结果、未运行项、diff review 和 reviewer owner；若要声称跨 agent 可发现/执行，必须增加至少一个受支持 agent 的 pinned-version behavioral eval。PR #6029 没有此类 agent eval，因此只建立 contributor guidance。^[PR #6029] + +## DOCSKILL-1c — Python example policy 只审新增路径并共享一份分类合同 + +- 触发：`precheck-pr` 或 `review-pr` 遇到在 `examples/` 下新增、复制或重命名的 Python + destination path。 +- 强制：从目标 revision 的 merge base 以 `ACR` path census 选出候选，再读取文件行为而不只看 + 文件名；model、checkpoint、vendor 或 family 专属的 prompt、request/output adaptation 与 launch + config 必须回 production model module 或 `model_extras`，用户命令和验证证据进 task docs/recipe。 + 两个 skill 必须引用同一份 canonical policy；shared image runner 的生产边界继续服从 + [EXEC-6a](../components/model-executor/rules.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。 +- 禁止：报告仅修改或删除的既有 model-specific example 债务；因文件名看似 generic 就接受内部只 + 实现一个模型合同的脚本；复制 policy 到 reviewer workflow 后分别演化；把 skill blocker 描述为 + 已有 CI gate。 +- 验收：分别覆盖 add、copy、rename、modify、delete，且 generic-looking model-specific 脚本必须 + block；真正 model-neutral、由配置选择模型的 task/protocol entrypoint 可通过。检查两个 skill 的 + reference 都解析到 canonical policy，并把结果作为独立 examples-policy 维度报告。^[PR #6046] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0a086f73e..780d52eb6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3c3989ed`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a15b5b4b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,8 +45,8 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `854bff7f` 四个 inventory 均无变化；PR #6074 只对齐 quantization overview 与 -导航中的 BitsAndBytes/GGUF 可发现性，不是 runtime registry 或支持范围变更。 +本次相对 `3c3989ed` 四个 inventory 均无变化；目标 revision 只修改 repository skill 的 +examples policy，不是 runtime registry 或支持范围变更。 ## AR/omni 模型族（29） @@ -89,8 +89,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 854bff7f \ - --to 3c3989ed \ + --from 3c3989ed \ + --to a15b5b4b \ --repo \ --mode report-only ``` From f7b2bf666d25776c4c6ef8bd7510ff066bfb5189 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 13:08:31 +0800 Subject: [PATCH 115/465] knowledge(vllm-omni): split diffusion and serving rules pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both owner rules pages were about to cross the 32768-byte hard split gate in check_knowledge_tree.py: diffusion/rules.md had 249 bytes of headroom and serving/rules.md 705, and both owners are touched frequently by the sync campaign. Splitting them now, deliberately, is much cheaper than having an unattended run discover the gate mid-unit and reach for the permanent 暂不拆分 waiver against the always-on briefing budget. Extractions follow the established rules-.md sibling pattern and keep every rule ID unchanged: diffusion DIFF-4a-4i -> rules-system-runtime.md serving SERV-5a-5e, SERV-6a-6d -> rules-engine-lifecycle.md Each new page is registered exactly once in the nearest _index.md, carries its own frontmatter and sources, and links back to the parent page and a sibling. Both parent pages route Direct reviewers to the extracted group, matching the existing SERV-5f pattern, and the two inbound SERV-5e anchors in configuration/adding-a-model.md and models/ming-flash-omni/_index.md now point at the extracted page. check_knowledge_tree only validates file existence, not anchors, so those were caught by test_knowledge_source.py instead. Headroom after: diffusion 12335 B, serving 10563 B. No unit is consumed and audited_sha is untouched, so the pin chain is unaffected. Gates: check_knowledge_tree 0 errors / 12 pre-existing reminders, check_wiki_lint 0 errors across 315 pages, release audit CLEAN, test_knowledge_source.py 12 passed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RAfw46exKmkdp1ytHsnhPn --- .../configuration/adding-a-model.md | 4 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-system-runtime.md | 157 ++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 146 +--------------- .../vllm-omni/components/serving/_index.md | 3 +- .../serving/rules-engine-lifecycle.md | 142 ++++++++++++++++ .../vllm-omni/components/serving/rules.md | 133 +-------------- .../models/ming-flash-omni/_index.md | 4 +- 8 files changed, 312 insertions(+), 280 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md diff --git a/knowledge/repos/vllm-omni/components/configuration/adding-a-model.md b/knowledge/repos/vllm-omni/components/configuration/adding-a-model.md index 9fdaa5dd2..423954eba 100644 --- a/knowledge/repos/vllm-omni/components/configuration/adding-a-model.md +++ b/knowledge/repos/vllm-omni/components/configuration/adding-a-model.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni 新模型接入路径" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-03 type: guide tags: [vllm-omni, components, config, models] sources: ["PR #5682", "claude-workflow-starter-private@296ea45", .claude/skills/add-tts-model/SKILL.md, docs/contributing/model/, vllm_omni/entrypoints/openai/tts_adapters/] @@ -34,7 +34,7 @@ AR TTS 新模型写一个 adapter，并在 `tts_adapters/__init__.py` 底部 imp 歧义/缺失时才声明 `model_archs`，只有无专有 stage 的 AR entry 才设 `arch_identifies_entry_stage`。非集合匹配用 `matches()`，部署拓扑能力用 `stage_serves_speech()`，真实冲突才设 `detect_priority`。详细 detection 不变量见 -[SERV-5e](../serving/rules.md#serv-5e-tts-detection-从-adapter-metadata-的有序并集派生)。纯 +[SERV-5e](../serving/rules-engine-lifecycle.md#serv-5e-tts-detection-从-adapter-metadata-的有序并集派生)。纯 diffusion TTS 仍走 `for_diffusion()` 直达路径，不能假设 adapter 的 `validate()`/`build()` 可达。 ## 照抄谁 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3e56b0173..66d34d53e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -1,7 +1,7 @@ --- title: "Diffusion" created: 2026-07-10 -updated: 2026-09-02 +updated: 2026-09-03 type: index tags: [vllm-omni, components, diffusion] sources: [] @@ -42,6 +42,7 @@ sources: [] | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | request-wave admission coalescing、stable window、deadline 与 finite config | [admission wait rules](rules-admission-wait.md) | +| paged KV/cache 预算、native/backend/platform 闭环、GQA/Ring/Ulysses layout、FlashInfer plan、能力 metadata | [paged cache 与系统运行时规则](rules-system-runtime.md) | | Scheduler-managed diffusion KV 的请求控制面、Hunyuan layout 与未实现边界 | [paged KV control plane](paged-kv-control-plane.md) | | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md new file mode 100644 index 000000000..4af2cb70b --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -0,0 +1,157 @@ +--- +title: "Diffusion paged cache 与系统运行时规则" +created: 2026-09-03 +updated: 2026-09-03 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py] +confidence: high +--- + +# Diffusion paged cache 与系统运行时规则 + +`system-runtime` 审查组的 `DIFF-4a`–`DIFF-4i`。触发条件、覆盖范围和其余审查组见 +[Diffusion 共享规则](rules.md) 的 Direct 代码快速入口；上游兼容侧的 `DIFF-4k`–`DIFF-4l` +留在 [upstream 兼容规则](rules-upstream-compat.md)。 + +## DIFF-4a — 容量预算必须在影响峰值的初始化完成后测量 + +- 触发：新增或修改 KV/cache block 数、GPU/CPU memory budget、auto-fit、warmup/profile、 + dummy run 或物理 cache 分配。 +- 强制：写出并核对 `model load → activation warmup/profile → available-memory measurement + → logical config → physical allocation → first real request` 的真实顺序；预算必须包含真实 + activation/graph/lazy-kernel 峰值及明确 headroom。若当前阶段故意只建逻辑控制面，必须使用 + 显式固定预算或把最终容量决定推迟到数据面 profile 之后。 +- 禁止：用 warmup 前的当前 free/process residency 推导可用 cache，再在之后执行会常驻或抬高 + 峰值的 dummy/profile；也不能用 native auto-fit 或 allocator 成功证明预算来源正确。 +- 验收：enabled 路径在真实初始化顺序下记录测量前后峰值，低显存/最大合法请求不会因漏算 + activation 在首次或后续执行 OOM；显式预算分支和自动预算分支各有测试。 ^[PR #6094] + +## DIFF-4b — 复用 native 组件仍须审计 caller/adapter 契约 + +- 触发：diffusion adapter 调用 vLLM 的 cache spec/config/manager、scheduler 或其他版本化 API。 +- 强制：按仓库 pin 的精确 vLLM 版本核对 property/method、签名、sentinel、block-size 单位、 + max-length/capacity 前提、返回失败语义和 free/rollback 责任；分别证明 adapter 输入、调用时序、 + 输出解释和 native 内部算法。 +- 禁止：把“委托给 native vLLM”当作端到端正确性证明；native manager 不负责验证 Omni 在何时 + 测量预算、是否漏算其他显存、是否正确传播失败，也不保证 adapter 模拟对象符合真实属性合同。 +- 验收：真实 pin 的 contract test 覆盖正常值、边界/sentinel、容量不足、property 访问和清理； + fake 不能把真实 property 改成同名 method 或跳过 native precondition。 ^[PR #6094] + +## DIFF-4c — Feature gate 只限制影响范围，enabled 路径必须独立闭环 + +- 触发：新增或修改 `dense_legacy`/paged、eager/optimized 或其他默认关闭的 diffusion 路径。 +- 强制：分别审计 disabled/default 与 enabled；enabled 至少覆盖单请求、并发/多 rank、低资源、 + partial allocation、执行异常、取消、timeout、shutdown 和重复释放。异常必须到达终端请求或 + engine 信号并唤醒 waiter，所有已分配资源由明确 owner 回收。 +- 禁止：用“默认路径有 gate”“dense 未回归”或单请求 smoke 推导新路径正确；也不能让 scheduler/ + busy-loop 异常逃逸后留下永远等待的 stream。 +- 验收：path/lifecycle matrix 每行有 production-path 测试或明确 `MISSING_EVIDENCE`；enabled + 关键行缺失时审查只能是 partial，不能输出 clean/no findings。 ^[PR #6094] + +## DIFF-4d — 异常清理覆盖每种错误传递形态 + +- 触发：worker/RPC 在异常后清 traceback、释放 device tensor/cache，或把异常转换成 + reply、rank-status envelope、`DiffusionOutput`。 +- 强制：先保存对外所需的字符串化错误，再对 raise、普通错误回复和 collect-status 正常返回 + 三条路径逐一执行同一清理；调用方把异常折叠成成功返回形态时仍必须释放原异常引用。 +- 禁止：只在最外层 `except` 清理，却让内部状态封装路径正常返回；清理后继续从异常对象读取 + message/traceback；用单一路径 mock 证明所有 RPC mode 都不会保留 device tensor。 +- 验收：测试分别覆盖 `collect_rank_status=true` 与抛出/错误回复路径，断言错误文本仍可见、 + `exc.__traceback__ is None`、cache cleanup 被调用，后续健康 RPC 仍成功。 ^[PR #6385] + +## DIFF-4e — paged backend 在启动时按平台和逐层能力闭环 + +- 触发：启用 paged scheduler、增加 diffusion attention backend 或平台 override。 +- 强制：启动阶段解析并验证每层实际 backend 与当前平台；走 Omni platform hook，保留 + piecewise/SP 等能力。尚无生产 caller 的模式必须明确告警并文档化。 +- 禁止：未知平台继承 GPU 默认；分配 cache 后到首次 dispatch 才失败；静默占用 paged KV + 却继续 dense；直接换用 upstream backend 丢失 Omni 能力。 +- 验收：unsupported platform/backend fail-fast；CUDA/NPU 对应 lane 和正式模型 E2E 证明 + resolved backend；mixed-mask piecewise/SP 回归通过。 ^[PR #5543] ^[PR #6102] + +## DIFF-4f — paged attention metadata 不从 padding 猜逻辑布局 + +- 触发：paged adapter 修改 GQA、prefix/current-token packing、LSE layout 或二维特例。 +- 强制：显式传递 Q/KV head 数、每请求 prefix/current span 和 layout；保持压缩 GQA 与 packed + current-token 语义，单 token/单 head 二维输入仍使用同一合同。 +- 禁止：从 padding 长度推断逻辑跨度；在 `sequence == heads` 时猜 LSE 维度；fallback 静默换 + backend 或把 KV heads 扩成 Q heads。 +- 验收：32Q/8KV、异构 prefix、单 token/单 head、非对称等维 LSE 和 batch compaction 精确 + 对照 dense/reference 输出。 ^[PR #5543] ^[PR #6102] + +## DIFF-4g — Ring GQA 只在通信后扩展，Hybrid Ulysses 不把 padding 当真 head + +- 触发：修改 Ring SDPA、GQA/MQA head 数、LSE accumulation、Ulysses all-to-all 或 + `advanced_uaa` head padding。 +- 强制：Ring 始终以原始 `H_kv` 通信 K/V；仅在本 rank 的 aten SDPA 调用前，当 + `H_q != H_kv` 时要求 `H_q % H_kv == 0`，再用 `repeat_interleave(H_q/H_kv)` 将 K/V 扩成 + Q head 数。该 local materialization 增加 kernel 输入计算/显存，但不增加 ring 通信量；不能 + 提前到 send/recv 前。之所以不能直接用 `enable_gqa`，是该 path 需要 aten flash/efficient + SDPA 返回 LSE 供 ring accumulation，而这些调用没有该参数。实现以 K head 数计算 factor 并 + 同样 repeat V，却未显式检查 `H_v == H_k`；现有测试始终令两者相等，future caller/backend 必须 + 保持或提前断言该 invariant，不能等 SDPA 在更深处 cryptic failure。 +- 强制：`advanced_uaa` + Hybrid Ulysses/ Ring 时，在 Ulysses all-to-all 前检查 key 和 value + 原始 head 数均可被 `ulysses_degree` 整除；否则 fail-fast。padding 出来的 zero K/V head 若交给 + Ring 再 replicate 会被当成真实 group，不能作为合法 MQA/GQA。pure Ring（Ulysses=1）仍支持 + 1 或多 KV heads；valid Hybrid positive fixture 必须使用可整除布局。 +- 禁止：用默认 backend 的绿测证明 SDPA 修复（安装 FA/FA3 时会绕过）；用 num_heads=3、 + Ulysses=2 的 padded case 当 Hybrid 正向测试；从通信量不变推断端到端性能不变。 +- 验收：4-card test 显式设 `TORCH_SDPA`，以 single-rank baseline 对 pure Ring GQA(8Q/2KV)、 + MQA(8Q/1KV) 和既有 Ulysses+Ring/AllGather-KV 输出，BF16 容差 5e-2；CPU guard 覆盖 Hybrid + padded-MQA 拒绝，另以 4Q heads、Ulysses2×Ring2 验证 valid Hybrid。ready CI 的 L4×4 job + 已收集 `test_attention_sp.py`，但不收集 `test_ulysses_uaa.py`；PR comment 的 primary matrix + 是 pre-final head `e6857888` 在 4×B300 上 4 passed、max abs 0.015625，runtime fix 已在但 final + target 未复测。修正为 4 heads 的 Hybrid 25.30s pass 只见 final PR body，未附 hardware/SHA/log。 + 因此目标 pin 有局部数值证据和未来 L4 gate wiring，但没有附带 L4 run 产物，Hybrid + guard/positive 也未被该新 job 覆盖。^[PR #5255] + +## DIFF-4h — FlashInfer mixed-dtype plan 必须绑定完整 runtime shape 与 mask 内容 + +- 触发：修改 `FLASHINFER_ATTN`、`BatchPrefillWithRaggedKVCacheWrapper`、QK16/V8、backend + selection、ragged indptr/plan cache 或 custom mask fallback。 +- 强制：输入保持初始化时 CUDA device；Q/K 只允许 fp16/bf16 override，V 另允许 + fp8_e4m3，wrapper 输出恢复原 query dtype。auto backend 按 compute capability 选 major>=10 + `cute-dsl`、major>=9 `fa3`、否则 `fa2`；non-CuTe 才接受 custom mask，CuTe nontrivial mask、 + causal+explicit mask 或无法 pack 的 mask 必须回退原始输入的 Torch SDPA。 +- 强制：plan key 至少绑定 batch、Q/KV length、Q/KV heads、QK/K/VO head dim、Q/K/V/output + dtype、causal、scale 与是否有 mask；shape/dtype 改变重建 indptr/plan，unmasked 同 key 可复用。 + mask shape 相同也可能内容不同，所以每次 masked call 都必须重新 plan，不能只按 pointer/shape + cache。Q/K/V cast 留在 compile-visible path，wrapper plan/run 边界才 disable compiler。目标 key + 虽记录 `head_dim_k`，plan 只接收 `head_dim_qk` 与 `head_dim_vo`，且未断言 Q/K head dim 相等； + caller 必须保持 `D_q == D_k`，并补 mismatch fail-fast test，不能把 key 字段误当 backend 已消费。 +- 禁止：把本 PR 早期 Sage/TRTLLM kernels 当成 merged scope（review 后删除并转交 TRTLLM + backend）；把 activation 直接 cast 成 FP8 描述成带 scale 的 checkpoint quantization；用 + `supports_attention_mask=True` 推断 CuTe 支持任意 mask。 +- 验收：mixed QK/V dtype 只能在 FlashInfer >=0.6.16rc1；目标对可解析的 <=0.6.15 fail-fast， + 但没有 dependency pin，只有 docs/code gate。该 gate 比较配置 override：两者均为 None 时立即 + 返回，即使 caller 实际传入 mixed Q/K 与 V tensor 也不拦；capability 必须最终按 plan 中真实 + q/k/v dtype 校验。缺失/非法 `flashinfer.__version__` 也会直接放行，是未封闭 gap。须补测试覆盖 + dtype allowlist、old/missing version、auto backend、device mismatch、plan reuse/replan、同 shape + 不同 mask、CuTe/causal fallback、GQA 与 output dtype；本 PR 没有新增 repo test，只有外部 smoke。 + PR 的 Cosmos3 1280×720、35 steps、seed42，B200/B300 DiT timing 与 Nano 前24帧 LPIPS=0.051496 + 只绑定 vLLM-Omni `4ce23c33` + FlashInfer `d0889c7c` 的 pre-final 环境；final config/version gate + commit 后无复测，且这些不是端到端 latency、稳定 speedup 或通用质量阈值。^[PR #5344] + +## DIFF-4i — 模型能力 metadata 必须跨 architecture 与 pipeline 两个 key space 解析 + +- 触发：修改 diffusion registry、HF `architectures`、`model_class_name` 或共享 + `DiffusionModelMetadata`，尤其是多图数量、mixed-reference 与 attention-mask-free 能力。 +- 强制：先接受 metadata 表中的 direct pipeline key；未命中时只通过 registry 的 + `architecture → pipeline class` 映射解析一次，再查 metadata。unknown architecture、`None`，以及 + registry 映射到没有 metadata 的 pipeline 都保持空 defaults；registry tuple 用解包读取，使布局 + 变化显式失败。此 helper 返回整份 metadata，影响 config、AsyncOmni serving view、视频 mixed + reference 和 attention backend，不得把修改只当作 image-count 局部修复。 +- 禁止：因 architecture 已注册就假定其 pipeline 有 metadata；递归猜测别名；让 direct key 被 + registry 映射覆盖；或把 defaults 的 `supports_multimodal_inputs=false` 当成“数量未知”——image-edit + admission 会把它解释为最多一张图，而 `supports=true, max=None` 才是不设上限。 +- 验收：同时覆盖 direct pipeline key、architecture→metadata-bearing pipeline、unknown/`None`、 + 以及 architecture→无 metadata pipeline；通过 `OmniDiffusionConfig.update_multimodal_support()` + 断言整份能力传播，并在 serving 边界验证 false→1、true+整数上限与无上限行为。PR #5838 只新增 + Qwen direct 与 Hunyuan architecture→pipeline 的手动 config 测试；未执行真实 architecture 解析、 + AsyncOmni 或 HTTP edit 路径，也没有 registry/metadata census 和 negative alias 回归。其 E2E 结果 + 只有截图，缺命令文本日志、vLLM 版本、upstream SHA 与重复次数。helper 仍 function-local import + 私有 `_DIFFUSION_MODELS`，review 提议公开改名但 final 未采用。^[PR #5838] + +相关执行流见 [Diffusion architecture](architecture.md)；paged attention 的 layout 细节见 +[attention 规则](rules-attention.md)。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 688ed9c0c..6ce2d6d29 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -1,7 +1,7 @@ --- title: "Diffusion 共享规则" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-03 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py] @@ -41,7 +41,7 @@ confidence: high | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2n` | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j` | +| `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j`（`4a`–`4i` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)） | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | ## 优化路径与 eager 的等价合同 @@ -214,147 +214,5 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 验收：运行测试文件中的 exact case，并在规则/配置旁保留最短资源原因；对称 baseline 证明质量差异来自目标量化变量。低 steps smoke 只能证明 runtime compatibility，不得声称 BF16 quality parity。 ^[PR #5136] ^[PR #6279] - -## Paged cache、资源预算与系统运行时 - -### DIFF-4a — 容量预算必须在影响峰值的初始化完成后测量 - -- 触发：新增或修改 KV/cache block 数、GPU/CPU memory budget、auto-fit、warmup/profile、 - dummy run 或物理 cache 分配。 -- 强制：写出并核对 `model load → activation warmup/profile → available-memory measurement - → logical config → physical allocation → first real request` 的真实顺序；预算必须包含真实 - activation/graph/lazy-kernel 峰值及明确 headroom。若当前阶段故意只建逻辑控制面，必须使用 - 显式固定预算或把最终容量决定推迟到数据面 profile 之后。 -- 禁止：用 warmup 前的当前 free/process residency 推导可用 cache，再在之后执行会常驻或抬高 - 峰值的 dummy/profile；也不能用 native auto-fit 或 allocator 成功证明预算来源正确。 -- 验收：enabled 路径在真实初始化顺序下记录测量前后峰值，低显存/最大合法请求不会因漏算 - activation 在首次或后续执行 OOM；显式预算分支和自动预算分支各有测试。 ^[PR #6094] - -### DIFF-4b — 复用 native 组件仍须审计 caller/adapter 契约 - -- 触发：diffusion adapter 调用 vLLM 的 cache spec/config/manager、scheduler 或其他版本化 API。 -- 强制：按仓库 pin 的精确 vLLM 版本核对 property/method、签名、sentinel、block-size 单位、 - max-length/capacity 前提、返回失败语义和 free/rollback 责任；分别证明 adapter 输入、调用时序、 - 输出解释和 native 内部算法。 -- 禁止：把“委托给 native vLLM”当作端到端正确性证明；native manager 不负责验证 Omni 在何时 - 测量预算、是否漏算其他显存、是否正确传播失败，也不保证 adapter 模拟对象符合真实属性合同。 -- 验收：真实 pin 的 contract test 覆盖正常值、边界/sentinel、容量不足、property 访问和清理； - fake 不能把真实 property 改成同名 method 或跳过 native precondition。 ^[PR #6094] - -### DIFF-4c — Feature gate 只限制影响范围，enabled 路径必须独立闭环 - -- 触发：新增或修改 `dense_legacy`/paged、eager/optimized 或其他默认关闭的 diffusion 路径。 -- 强制：分别审计 disabled/default 与 enabled；enabled 至少覆盖单请求、并发/多 rank、低资源、 - partial allocation、执行异常、取消、timeout、shutdown 和重复释放。异常必须到达终端请求或 - engine 信号并唤醒 waiter，所有已分配资源由明确 owner 回收。 -- 禁止：用“默认路径有 gate”“dense 未回归”或单请求 smoke 推导新路径正确；也不能让 scheduler/ - busy-loop 异常逃逸后留下永远等待的 stream。 -- 验收：path/lifecycle matrix 每行有 production-path 测试或明确 `MISSING_EVIDENCE`；enabled - 关键行缺失时审查只能是 partial，不能输出 clean/no findings。 ^[PR #6094] - -### DIFF-4d — 异常清理覆盖每种错误传递形态 - -- 触发：worker/RPC 在异常后清 traceback、释放 device tensor/cache，或把异常转换成 - reply、rank-status envelope、`DiffusionOutput`。 -- 强制：先保存对外所需的字符串化错误，再对 raise、普通错误回复和 collect-status 正常返回 - 三条路径逐一执行同一清理；调用方把异常折叠成成功返回形态时仍必须释放原异常引用。 -- 禁止：只在最外层 `except` 清理，却让内部状态封装路径正常返回；清理后继续从异常对象读取 - message/traceback；用单一路径 mock 证明所有 RPC mode 都不会保留 device tensor。 -- 验收：测试分别覆盖 `collect_rank_status=true` 与抛出/错误回复路径，断言错误文本仍可见、 - `exc.__traceback__ is None`、cache cleanup 被调用，后续健康 RPC 仍成功。 ^[PR #6385] - -### DIFF-4e — paged backend 在启动时按平台和逐层能力闭环 - -- 触发：启用 paged scheduler、增加 diffusion attention backend 或平台 override。 -- 强制：启动阶段解析并验证每层实际 backend 与当前平台；走 Omni platform hook，保留 - piecewise/SP 等能力。尚无生产 caller 的模式必须明确告警并文档化。 -- 禁止：未知平台继承 GPU 默认；分配 cache 后到首次 dispatch 才失败；静默占用 paged KV - 却继续 dense；直接换用 upstream backend 丢失 Omni 能力。 -- 验收：unsupported platform/backend fail-fast；CUDA/NPU 对应 lane 和正式模型 E2E 证明 - resolved backend；mixed-mask piecewise/SP 回归通过。 ^[PR #5543] ^[PR #6102] - -### DIFF-4f — paged attention metadata 不从 padding 猜逻辑布局 - -- 触发：paged adapter 修改 GQA、prefix/current-token packing、LSE layout 或二维特例。 -- 强制：显式传递 Q/KV head 数、每请求 prefix/current span 和 layout；保持压缩 GQA 与 packed - current-token 语义，单 token/单 head 二维输入仍使用同一合同。 -- 禁止：从 padding 长度推断逻辑跨度；在 `sequence == heads` 时猜 LSE 维度；fallback 静默换 - backend 或把 KV heads 扩成 Q heads。 -- 验收：32Q/8KV、异构 prefix、单 token/单 head、非对称等维 LSE 和 batch compaction 精确 - 对照 dense/reference 输出。 ^[PR #5543] ^[PR #6102] - -### DIFF-4g — Ring GQA 只在通信后扩展，Hybrid Ulysses 不把 padding 当真 head - -- 触发：修改 Ring SDPA、GQA/MQA head 数、LSE accumulation、Ulysses all-to-all 或 - `advanced_uaa` head padding。 -- 强制：Ring 始终以原始 `H_kv` 通信 K/V；仅在本 rank 的 aten SDPA 调用前，当 - `H_q != H_kv` 时要求 `H_q % H_kv == 0`，再用 `repeat_interleave(H_q/H_kv)` 将 K/V 扩成 - Q head 数。该 local materialization 增加 kernel 输入计算/显存，但不增加 ring 通信量；不能 - 提前到 send/recv 前。之所以不能直接用 `enable_gqa`，是该 path 需要 aten flash/efficient - SDPA 返回 LSE 供 ring accumulation，而这些调用没有该参数。实现以 K head 数计算 factor 并 - 同样 repeat V，却未显式检查 `H_v == H_k`；现有测试始终令两者相等，future caller/backend 必须 - 保持或提前断言该 invariant，不能等 SDPA 在更深处 cryptic failure。 -- 强制：`advanced_uaa` + Hybrid Ulysses/ Ring 时，在 Ulysses all-to-all 前检查 key 和 value - 原始 head 数均可被 `ulysses_degree` 整除；否则 fail-fast。padding 出来的 zero K/V head 若交给 - Ring 再 replicate 会被当成真实 group，不能作为合法 MQA/GQA。pure Ring（Ulysses=1）仍支持 - 1 或多 KV heads；valid Hybrid positive fixture 必须使用可整除布局。 -- 禁止：用默认 backend 的绿测证明 SDPA 修复（安装 FA/FA3 时会绕过）；用 num_heads=3、 - Ulysses=2 的 padded case 当 Hybrid 正向测试；从通信量不变推断端到端性能不变。 -- 验收：4-card test 显式设 `TORCH_SDPA`，以 single-rank baseline 对 pure Ring GQA(8Q/2KV)、 - MQA(8Q/1KV) 和既有 Ulysses+Ring/AllGather-KV 输出，BF16 容差 5e-2；CPU guard 覆盖 Hybrid - padded-MQA 拒绝，另以 4Q heads、Ulysses2×Ring2 验证 valid Hybrid。ready CI 的 L4×4 job - 已收集 `test_attention_sp.py`，但不收集 `test_ulysses_uaa.py`；PR comment 的 primary matrix - 是 pre-final head `e6857888` 在 4×B300 上 4 passed、max abs 0.015625，runtime fix 已在但 final - target 未复测。修正为 4 heads 的 Hybrid 25.30s pass 只见 final PR body，未附 hardware/SHA/log。 - 因此目标 pin 有局部数值证据和未来 L4 gate wiring，但没有附带 L4 run 产物，Hybrid - guard/positive 也未被该新 job 覆盖。^[PR #5255] - -### DIFF-4h — FlashInfer mixed-dtype plan 必须绑定完整 runtime shape 与 mask 内容 - -- 触发：修改 `FLASHINFER_ATTN`、`BatchPrefillWithRaggedKVCacheWrapper`、QK16/V8、backend - selection、ragged indptr/plan cache 或 custom mask fallback。 -- 强制：输入保持初始化时 CUDA device；Q/K 只允许 fp16/bf16 override，V 另允许 - fp8_e4m3，wrapper 输出恢复原 query dtype。auto backend 按 compute capability 选 major>=10 - `cute-dsl`、major>=9 `fa3`、否则 `fa2`；non-CuTe 才接受 custom mask，CuTe nontrivial mask、 - causal+explicit mask 或无法 pack 的 mask 必须回退原始输入的 Torch SDPA。 -- 强制：plan key 至少绑定 batch、Q/KV length、Q/KV heads、QK/K/VO head dim、Q/K/V/output - dtype、causal、scale 与是否有 mask；shape/dtype 改变重建 indptr/plan，unmasked 同 key 可复用。 - mask shape 相同也可能内容不同，所以每次 masked call 都必须重新 plan，不能只按 pointer/shape - cache。Q/K/V cast 留在 compile-visible path，wrapper plan/run 边界才 disable compiler。目标 key - 虽记录 `head_dim_k`，plan 只接收 `head_dim_qk` 与 `head_dim_vo`，且未断言 Q/K head dim 相等； - caller 必须保持 `D_q == D_k`，并补 mismatch fail-fast test，不能把 key 字段误当 backend 已消费。 -- 禁止：把本 PR 早期 Sage/TRTLLM kernels 当成 merged scope（review 后删除并转交 TRTLLM - backend）；把 activation 直接 cast 成 FP8 描述成带 scale 的 checkpoint quantization；用 - `supports_attention_mask=True` 推断 CuTe 支持任意 mask。 -- 验收：mixed QK/V dtype 只能在 FlashInfer >=0.6.16rc1；目标对可解析的 <=0.6.15 fail-fast， - 但没有 dependency pin，只有 docs/code gate。该 gate 比较配置 override：两者均为 None 时立即 - 返回，即使 caller 实际传入 mixed Q/K 与 V tensor 也不拦；capability 必须最终按 plan 中真实 - q/k/v dtype 校验。缺失/非法 `flashinfer.__version__` 也会直接放行，是未封闭 gap。须补测试覆盖 - dtype allowlist、old/missing version、auto backend、device mismatch、plan reuse/replan、同 shape - 不同 mask、CuTe/causal fallback、GQA 与 output dtype；本 PR 没有新增 repo test，只有外部 smoke。 - PR 的 Cosmos3 1280×720、35 steps、seed42，B200/B300 DiT timing 与 Nano 前24帧 LPIPS=0.051496 - 只绑定 vLLM-Omni `4ce23c33` + FlashInfer `d0889c7c` 的 pre-final 环境；final config/version gate - commit 后无复测，且这些不是端到端 latency、稳定 speedup 或通用质量阈值。^[PR #5344] - -### DIFF-4i — 模型能力 metadata 必须跨 architecture 与 pipeline 两个 key space 解析 - -- 触发：修改 diffusion registry、HF `architectures`、`model_class_name` 或共享 - `DiffusionModelMetadata`，尤其是多图数量、mixed-reference 与 attention-mask-free 能力。 -- 强制：先接受 metadata 表中的 direct pipeline key；未命中时只通过 registry 的 - `architecture → pipeline class` 映射解析一次，再查 metadata。unknown architecture、`None`，以及 - registry 映射到没有 metadata 的 pipeline 都保持空 defaults；registry tuple 用解包读取，使布局 - 变化显式失败。此 helper 返回整份 metadata，影响 config、AsyncOmni serving view、视频 mixed - reference 和 attention backend，不得把修改只当作 image-count 局部修复。 -- 禁止：因 architecture 已注册就假定其 pipeline 有 metadata；递归猜测别名；让 direct key 被 - registry 映射覆盖；或把 defaults 的 `supports_multimodal_inputs=false` 当成“数量未知”——image-edit - admission 会把它解释为最多一张图，而 `supports=true, max=None` 才是不设上限。 -- 验收：同时覆盖 direct pipeline key、architecture→metadata-bearing pipeline、unknown/`None`、 - 以及 architecture→无 metadata pipeline；通过 `OmniDiffusionConfig.update_multimodal_support()` - 断言整份能力传播，并在 serving 边界验证 false→1、true+整数上限与无上限行为。PR #5838 只新增 - Qwen direct 与 Hunyuan architecture→pipeline 的手动 config 测试；未执行真实 architecture 解析、 - AsyncOmni 或 HTTP edit 路径，也没有 registry/metadata census 和 negative alias 回归。其 E2E 结果 - 只有截图，缺命令文本日志、vLLM 版本、upstream SHA 与重复次数。helper 仍 function-local import - 私有 `_DIFFUSION_MODELS`，review 提议公开改名但 final 未采用。^[PR #5838] - 相关执行流见 [Diffusion architecture](architecture.md)；benchmark 证据合同见 [performance evidence](../../benchmark/guides/performance-evidence.md)。 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 08e75401f..6febbbfee 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -1,7 +1,7 @@ --- title: "Serving" created: 2026-07-10 -updated: 2026-08-23 +updated: 2026-09-03 type: index tags: [vllm-omni, components, serving] sources: [] @@ -32,3 +32,4 @@ sources: [] | upstream launcher shutdown、renderer warmup 与多 replica compile-cache 隔离 | [upstream 兼容规则](rules-upstream-compat.md) | | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | +| engine startup/shutdown 顺序、stage 生命周期、full-duplex 与 CFG companion | [engine 生命周期规则](rules-engine-lifecycle.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md new file mode 100644 index 000000000..5e13c3b02 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -0,0 +1,142 @@ +--- +title: "Serving engine 生命周期规则" +created: 2026-09-03 +updated: 2026-09-03 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py] +confidence: high +--- + +# Serving engine 生命周期规则 + +`SERV-5a`–`SERV-5e` 与 `SERV-6a`–`SERV-6d`。触发条件与其余审查组见 +[Serving 共享规则](rules.md) 的 Direct 代码快速入口；请求输入侧的合同留在该页， +故障隔离见 [fault isolation 规则](rules-fault-isolation.md)。 + +## Engine 生命周期合同 + +### SERV-5a — sleep/wake 状态必须保留 stage 和 tag 作用域 + +- 触发：sleep/wake 接受 `stage_ids`、resource tags 或 partial wake。 +- 强制：状态 key 与公开操作的 stage/tag 作用域一致；只有全部必需 stage/tag 已 warm + 才放行 generation。 +- 禁止：用一个全局 tag set 表示多 stage 状态；唤醒一个 stage 后清掉其他 stage 的 + sleeping 状态或把后续定向 wake 当成 already warm。 +- 验收：sleep 两个 stage、只 wake 一个时 generation 仍拒绝，随后 wake 另一个才放行。 + ^[PR #4834] + +### SERV-5b — 只有成功 ACK 和真实 backend capability 才能转为 warm + +- 触发：worker ACK 可返回 error，或不同 backend 对 level-2 restore 能力不同。 +- 强制：逐目标确认成功 ACK 后再清状态；level-2 能力按 backend/stage 表达。 +- 禁止：错误 ACK 也清 tag；用 engine 全局禁令误伤已经支持 restore 的 diffusion worker。 +- 验收：失败 ACK 保留 sleeping 状态；支持 level-2 的 diffusion 路径仍能 + sleep → wake → generate，不支持的 stage 在调用 worker 前明确拒绝。 + assembled app 对不支持的 level-2 wake 保留 sleeping state，并把既有 + `NotImplementedError` 映射为 OpenAI-style structured HTTP 501；bare route mock 只证明 + exception propagation，不能代替这条 live contract。^[PR #4834] ^[PR #4905] + ^[PR #4912] ^[PR #5713] + +### SERV-5c — 共享服务类重构必须闭合全部构造状态 + +- 触发：移动 model-specific capability 到 adapter、增加 factory/`__new__` 快路径，或让同一 + serving class 同时承载普通、无对应 stage、diffusion-only 等实例。 +- 强制：列出每个生产构造入口及其必有/可选属性；所有公共 caller 通过统一 accessor 或显式 + `None` 状态访问 optional capability，startup warmup 与 request route 使用同一状态合同。 +- 禁止：只测试目标 adapter 的正常构造；在 factory 实例上直接读取未初始化属性；用旧的 + model-type early return 掩盖无 adapter 状态。 +- 验收：至少覆盖普通 adapter、无 adapter 和 bypass-`__init__` factory 三类实例，并实际调用 + startup warmup、voice list/upload 与对应 request route；每条路径保持重构前的成功或结构化 + 错误合同，不得出现 `AttributeError`。 ^[PR #6138] + +### SERV-5d — 应用 wiring guard 检查可用性而非属性名 + +- 触发：assembled app 的 route/middleware 依赖 `app.state` handler 或 factory wiring。 +- 强制：mandatory state 同时检查属性存在且非 `None`；guard 测试通过 assembled app/公开 handler + 观察行为，并覆盖应用实际暴露的方法和 middleware inner-app 调用。 +- 禁止：`hasattr` 接受显式 `None`；测试绑定可搬迁私有 symbol/单 router；用恒真断言证明 wiring。 +- 验收：把 handler 置空、移除 route/method、绕过 inner app 三类 mutation 均使 guard 失败； + 正常 app 的 HEAD/OPTIONS census 与必需 state key 完整。 ^[PR #6202] + +### SERV-5e — TTS detection 从 adapter metadata 的有序并集派生 + +- 触发：增加/迁移 TTS adapter，修改 `stage_keys`、`model_archs`、stage discovery、检测顺序 + 或 speech-capable deployment topology。 +- 强制：adapter 与显式 `LEGACY_TTS_DETECTORS` 构成完整 detector 并集；按 + `(detect_priority, name)` 确定性排序，等优先级 detector 不得重叠。stage-key 候选、仅靠 + architecture 定位的 AR entry stage、最终 model type 都从该并集派生；非集合规则覆写 + `matches()`，拓扑能力覆写 `stage_serves_speech()`，不要回填 `serving_speech.py` 分支。 +- 边界：VoxCPM2 talker architecture 是高优先级权威匹配；Ming flash adapter 的 `ming_tts` + stage-key 以默认 priority 100 先于 Ming dense 的 architecture fallback priority 200；CoVo 的通用 `fused_thinker_talker` 还需 + architecture 确认；Audex omni 还需同部署的 `audex_code2wav`。纯 diffusion 的 + `for_diffusion()` 直接绕过 adapter detection，当前 `DiffusionTTSAdapter` 没有生产 subclass。 +- 验收：冻结旧 ladder 作为 oracle，覆盖 in-tree pipeline stage、adapter 声明 architecture、 + stage 集合等价、entry-arch 范围、无歧义/稳定顺序和 topology guard。`LegacyDetector` 只能缩减； + AST ratchet 中超过 branch/legacy budget 失败，等于上限静默通过，少于上限则通过 + 并显示收紧 reminder；下降本身不能让该 pre-commit hook 红灯，但未同步更新的 exact-equality + pytest 仍会失败。在人工调低 constant 前存在可回增 slack，table dispatch 也是已固定的漏检形态， + 不能把 pre-commit 通过解释为不可绕过。 + `omnivoice_generator` 未被 pipeline 发出，是保留兼容事实而非 + 可达性证据。^[PR #5682] +- adapter 提取保持共享 dispatcher 单一：Ming Flash 的 `validate()` 原样拒绝空 input、过长/非 + 字符串 instructions、`task_type`/`language`/`x_vector_only_mode`/ + `initial_codec_chunk_frames`、`ref_audio`/`ref_text` 和非正 `max_new_tokens`；`build()` 继续委托 + server 的 `_build_ming_flash_omni_prompt()`，并返回空 `tts_params` 与固定 model type。公开校验 + 和 generation preparation 都必须经 `resolve_adapter()`，不要把同一分支加回 + `serving_speech.py`。`LEGACY_TTS_DETECTORS` 当前为空，branch/legacy AST ratchet 是 27/0； + Ming Flash 只由 adapter 的 `ming_tts` stage key 检测，且 legacy 名不得同时注册 adapter。 + 验收仍须证明 `ming_tts` stage key 与 Ming dense architecture fallback 的消歧不变。 + ^[PR #5746] ^[PR #5843] ^[PR #6008] +- 证据：CPU oracle/ratchet suite 支撑检测等价；L20X VoxCPM2 只证明 architecture path 到达 + warmup，随后因基线同样复现的 vLLM skew 退出，未证明成功 speech endpoint/audio E2E。 + +## Full-duplex 与 CFG companion 生命周期 + +### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology + +- 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 +- 强制：stage-0 的首次 `submit_initial` 在 async-chunk 开启时预热后续 stage 的 + runtime/pool；每个 stage 保留自己的 fence、submit timestamp 和 request state。 +- 禁止：等到 duplex audio/control 事件真正抵达才首次启动下游 stage；用一个全局 fence + 表示多 stage readiness；prewarm 失败后仍发布 generation-ready。 +- 验收：覆盖首次 stage-0 submit、后续 stage submit、重复 update 和 prewarm failure； + CPU/mock 测试必须证明 request state 与 stage pool 生命周期一致。 + +### SERV-6b — CFG companion 输出要非破坏性聚合并在所有 teardown 路径清理 + +- 触发：CFG companion、deferred parent、streaming terminal update、abort、stage error + 或 replica loss。 +- 强制：parent re-submit 前 companion outputs 可重复读取；parent 只在全部 companion + 完成后释放；companion error/abort 必须给 deferred parent 发送结构化错误，并由统一 + cleanup 路径清理 parent、companion、tracker 和 stage-pool binding。 +- 禁止：用 destructive `pop` 让第二次 terminal update 丢 companion；只清 parent 不清 + companion；companion 永久等待时静默挂起 parent。 +- 验收：覆盖正常全量完成、重复读取、companion error、parent abort、companion abort + 和 replica loss，断言没有遗留 tracker state 或未完成的 deferred parent。 + +### SERV-6c — duplex capability gate 必须区分未启用与配置失败 + +- 触发：`session_mode=duplex`、`/v1/realtime?duplex=true`、runtime extension 或 + duplex deploy configuration 发生变化。 +- 强制：入口只在明确的 duplex opt-in 下建立 handler；能力检查必须保留结构化配置 + 错误与“不支持该模式”的区别，session、request、stage fence、lease 和 ordered + mailbox 的生命周期必须在 disconnect、abort、expiry 与 replica loss 时一起终止。 +- 禁止：把所有 config-load exception 折叠成 `duplex unavailable`；用普通 streaming + handler 冒充持久 session；用全局 fence 或无界 pending input 替代 per-session bound。 +- 验收：覆盖未 opt-in、有效 opt-in、malformed config、stale fence、lease expiry、 + disconnect 和 terminal response ordering；控制消息中的 dataclass/enum/tensor 必须 + 在发送前变成 JSON-safe 值。 + +请求到 engine 的边界见 [Serving architecture](architecture.md)；公开协议通用检查见 +[review contracts](../../../../general/review/guides/reviewer-lens-contracts.md)。 + +### SERV-6d — tokenizer-free 与 native speed 由 adapter capability 闭环 + +- 触发：stage 0 跳过 tokenizer，或 TTS adapter 声明 native speed control。 +- 强制：token-only renderer 仅接受 token IDs/prompt embeddings，chat messages 明确失败。HTTP raw、SSE 与 + non-streaming 由 adapter 校验 native speed，encoder speed 固定 1；generic model 拒绝 `speed != 1`， + WebSocket 在 parity 完成前固定 speed=1。 +- 禁止：模型 duration scaling 后再次 resample，或以 HTTP 支持外推 WebSocket parity。 +- 验收：覆盖 token/chat/embed、native/generic、三种 HTTP 输出与 WebSocket。落后后端只用显式 + capability/version shim 并保留 eager fallback，不能让 CUDA 新参数成为全平台前提。^[PR #5957] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 9108e8e08..62dc19c77 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -1,7 +1,7 @@ --- title: "Serving 规则" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-03 type: rule tags: [vllm-omni, components, serving] sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] @@ -42,8 +42,8 @@ confidence: high | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c` + 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d`, `SERV-5d` | -| `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection、replica fault isolation | `SERV-5a`, `SERV-5b`, `SERV-5c`, `SERV-5e`, `SERV-5f` | -| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6c` | +| `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection、replica fault isolation | `SERV-5a`–`SERV-5e`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | +| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`, `SERV-4b`, `SERV-4c`, `SERV-4d`, `SERV-4e`, `SERV-4f`, `SERV-4g`, `SERV-4h`, `SERV-4l` | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`, `SERV-4j`, `SERV-4k` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | @@ -254,130 +254,3 @@ confidence: high `sampling_params_list[i].lora_request` 拥有，默认参数对象不被污染。GPU 验收必须先确认 adapter ID 已加载，再证明 deterministic token/logprob 与 base 不同；#5369 另报的 `AsyncOmni.add_lora` control-RPC 反序列化成 list 问题不在本修复范围。^[PR #5374] - -## Engine 生命周期合同 - -### SERV-5a — sleep/wake 状态必须保留 stage 和 tag 作用域 - -- 触发：sleep/wake 接受 `stage_ids`、resource tags 或 partial wake。 -- 强制：状态 key 与公开操作的 stage/tag 作用域一致；只有全部必需 stage/tag 已 warm - 才放行 generation。 -- 禁止：用一个全局 tag set 表示多 stage 状态；唤醒一个 stage 后清掉其他 stage 的 - sleeping 状态或把后续定向 wake 当成 already warm。 -- 验收：sleep 两个 stage、只 wake 一个时 generation 仍拒绝，随后 wake 另一个才放行。 - ^[PR #4834] - -### SERV-5b — 只有成功 ACK 和真实 backend capability 才能转为 warm - -- 触发：worker ACK 可返回 error，或不同 backend 对 level-2 restore 能力不同。 -- 强制：逐目标确认成功 ACK 后再清状态；level-2 能力按 backend/stage 表达。 -- 禁止：错误 ACK 也清 tag；用 engine 全局禁令误伤已经支持 restore 的 diffusion worker。 -- 验收：失败 ACK 保留 sleeping 状态；支持 level-2 的 diffusion 路径仍能 - sleep → wake → generate，不支持的 stage 在调用 worker 前明确拒绝。 - assembled app 对不支持的 level-2 wake 保留 sleeping state，并把既有 - `NotImplementedError` 映射为 OpenAI-style structured HTTP 501；bare route mock 只证明 - exception propagation，不能代替这条 live contract。^[PR #4834] ^[PR #4905] - ^[PR #4912] ^[PR #5713] - -### SERV-5c — 共享服务类重构必须闭合全部构造状态 - -- 触发：移动 model-specific capability 到 adapter、增加 factory/`__new__` 快路径，或让同一 - serving class 同时承载普通、无对应 stage、diffusion-only 等实例。 -- 强制：列出每个生产构造入口及其必有/可选属性；所有公共 caller 通过统一 accessor 或显式 - `None` 状态访问 optional capability，startup warmup 与 request route 使用同一状态合同。 -- 禁止：只测试目标 adapter 的正常构造；在 factory 实例上直接读取未初始化属性；用旧的 - model-type early return 掩盖无 adapter 状态。 -- 验收：至少覆盖普通 adapter、无 adapter 和 bypass-`__init__` factory 三类实例，并实际调用 - startup warmup、voice list/upload 与对应 request route；每条路径保持重构前的成功或结构化 - 错误合同，不得出现 `AttributeError`。 ^[PR #6138] - -### SERV-5d — 应用 wiring guard 检查可用性而非属性名 - -- 触发：assembled app 的 route/middleware 依赖 `app.state` handler 或 factory wiring。 -- 强制：mandatory state 同时检查属性存在且非 `None`；guard 测试通过 assembled app/公开 handler - 观察行为，并覆盖应用实际暴露的方法和 middleware inner-app 调用。 -- 禁止：`hasattr` 接受显式 `None`；测试绑定可搬迁私有 symbol/单 router；用恒真断言证明 wiring。 -- 验收：把 handler 置空、移除 route/method、绕过 inner app 三类 mutation 均使 guard 失败； - 正常 app 的 HEAD/OPTIONS census 与必需 state key 完整。 ^[PR #6202] - -### SERV-5e — TTS detection 从 adapter metadata 的有序并集派生 - -- 触发：增加/迁移 TTS adapter，修改 `stage_keys`、`model_archs`、stage discovery、检测顺序 - 或 speech-capable deployment topology。 -- 强制：adapter 与显式 `LEGACY_TTS_DETECTORS` 构成完整 detector 并集；按 - `(detect_priority, name)` 确定性排序，等优先级 detector 不得重叠。stage-key 候选、仅靠 - architecture 定位的 AR entry stage、最终 model type 都从该并集派生；非集合规则覆写 - `matches()`，拓扑能力覆写 `stage_serves_speech()`，不要回填 `serving_speech.py` 分支。 -- 边界：VoxCPM2 talker architecture 是高优先级权威匹配；Ming flash adapter 的 `ming_tts` - stage-key 以默认 priority 100 先于 Ming dense 的 architecture fallback priority 200；CoVo 的通用 `fused_thinker_talker` 还需 - architecture 确认；Audex omni 还需同部署的 `audex_code2wav`。纯 diffusion 的 - `for_diffusion()` 直接绕过 adapter detection，当前 `DiffusionTTSAdapter` 没有生产 subclass。 -- 验收：冻结旧 ladder 作为 oracle，覆盖 in-tree pipeline stage、adapter 声明 architecture、 - stage 集合等价、entry-arch 范围、无歧义/稳定顺序和 topology guard。`LegacyDetector` 只能缩减； - AST ratchet 中超过 branch/legacy budget 失败，等于上限静默通过，少于上限则通过 - 并显示收紧 reminder；下降本身不能让该 pre-commit hook 红灯，但未同步更新的 exact-equality - pytest 仍会失败。在人工调低 constant 前存在可回增 slack，table dispatch 也是已固定的漏检形态， - 不能把 pre-commit 通过解释为不可绕过。 - `omnivoice_generator` 未被 pipeline 发出，是保留兼容事实而非 - 可达性证据。^[PR #5682] -- adapter 提取保持共享 dispatcher 单一：Ming Flash 的 `validate()` 原样拒绝空 input、过长/非 - 字符串 instructions、`task_type`/`language`/`x_vector_only_mode`/ - `initial_codec_chunk_frames`、`ref_audio`/`ref_text` 和非正 `max_new_tokens`；`build()` 继续委托 - server 的 `_build_ming_flash_omni_prompt()`，并返回空 `tts_params` 与固定 model type。公开校验 - 和 generation preparation 都必须经 `resolve_adapter()`，不要把同一分支加回 - `serving_speech.py`。`LEGACY_TTS_DETECTORS` 当前为空，branch/legacy AST ratchet 是 27/0； - Ming Flash 只由 adapter 的 `ming_tts` stage key 检测，且 legacy 名不得同时注册 adapter。 - 验收仍须证明 `ming_tts` stage key 与 Ming dense architecture fallback 的消歧不变。 - ^[PR #5746] ^[PR #5843] ^[PR #6008] -- 证据：CPU oracle/ratchet suite 支撑检测等价；L20X VoxCPM2 只证明 architecture path 到达 - warmup，随后因基线同样复现的 vLLM skew 退出，未证明成功 speech endpoint/audio E2E。 - -## Full-duplex 与 CFG companion 生命周期 - -### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - -- 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 -- 强制：stage-0 的首次 `submit_initial` 在 async-chunk 开启时预热后续 stage 的 - runtime/pool；每个 stage 保留自己的 fence、submit timestamp 和 request state。 -- 禁止：等到 duplex audio/control 事件真正抵达才首次启动下游 stage；用一个全局 fence - 表示多 stage readiness；prewarm 失败后仍发布 generation-ready。 -- 验收：覆盖首次 stage-0 submit、后续 stage submit、重复 update 和 prewarm failure； - CPU/mock 测试必须证明 request state 与 stage pool 生命周期一致。 - -### SERV-6b — CFG companion 输出要非破坏性聚合并在所有 teardown 路径清理 - -- 触发：CFG companion、deferred parent、streaming terminal update、abort、stage error - 或 replica loss。 -- 强制：parent re-submit 前 companion outputs 可重复读取；parent 只在全部 companion - 完成后释放；companion error/abort 必须给 deferred parent 发送结构化错误，并由统一 - cleanup 路径清理 parent、companion、tracker 和 stage-pool binding。 -- 禁止：用 destructive `pop` 让第二次 terminal update 丢 companion；只清 parent 不清 - companion；companion 永久等待时静默挂起 parent。 -- 验收：覆盖正常全量完成、重复读取、companion error、parent abort、companion abort - 和 replica loss，断言没有遗留 tracker state 或未完成的 deferred parent。 - -### SERV-6c — duplex capability gate 必须区分未启用与配置失败 - -- 触发：`session_mode=duplex`、`/v1/realtime?duplex=true`、runtime extension 或 - duplex deploy configuration 发生变化。 -- 强制：入口只在明确的 duplex opt-in 下建立 handler；能力检查必须保留结构化配置 - 错误与“不支持该模式”的区别，session、request、stage fence、lease 和 ordered - mailbox 的生命周期必须在 disconnect、abort、expiry 与 replica loss 时一起终止。 -- 禁止：把所有 config-load exception 折叠成 `duplex unavailable`；用普通 streaming - handler 冒充持久 session；用全局 fence 或无界 pending input 替代 per-session bound。 -- 验收：覆盖未 opt-in、有效 opt-in、malformed config、stale fence、lease expiry、 - disconnect 和 terminal response ordering；控制消息中的 dataclass/enum/tensor 必须 - 在发送前变成 JSON-safe 值。 - -请求到 engine 的边界见 [Serving architecture](architecture.md)；公开协议通用检查见 -[review contracts](../../../../general/review/guides/reviewer-lens-contracts.md)。 - -### SERV-6d — tokenizer-free 与 native speed 由 adapter capability 闭环 - -- 触发：stage 0 跳过 tokenizer，或 TTS adapter 声明 native speed control。 -- 强制：token-only renderer 仅接受 token IDs/prompt embeddings，chat messages 明确失败。HTTP raw、SSE 与 - non-streaming 由 adapter 校验 native speed，encoder speed 固定 1；generic model 拒绝 `speed != 1`， - WebSocket 在 parity 完成前固定 speed=1。 -- 禁止：模型 duration scaling 后再次 resample，或以 HTTP 支持外推 WebSocket parity。 -- 验收：覆盖 token/chat/embed、native/generic、三种 HTTP 输出与 WebSocket。落后后端只用显式 - capability/version shim 并保留 eager fallback，不能让 CUDA 新参数成为全平台前提。^[PR #5957] diff --git a/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md b/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md index 9149804c1..0e2540aa8 100644 --- a/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/ming-flash-omni/_index.md @@ -1,7 +1,7 @@ --- title: "Ming-flash-omni（BailingMM2,4 拓扑全模态）" created: 2026-07-21 -updated: 2026-08-10 +updated: 2026-09-03 type: index tags: [vllm-omni, models, diffusion] sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/models/ming_flash_omni/, vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py, vllm_omni/entrypoints/openai/tts_adapters/__init__.py, vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py, tools/pre_commit/check_tts_adapter.py, "PR #5746", "PR #5843"] @@ -43,7 +43,7 @@ sources: [vllm_omni/model_executor/models/ming_flash_omni/, vllm_omni/diffusion/ | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | 文本桥 vs hidden 桥、CFG 伴随、ByT5 字形路径 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | -| speech API 的 detection/adapter union | [Serving SERV-5e](../../components/serving/rules.md#serv-5e-tts-detection-从-adapter-metadata-的有序并集派生) | `ming_tts` 消歧、请求校验和 prompt delegation | +| speech API 的 detection/adapter union | [Serving SERV-5e](../../components/serving/rules-engine-lifecycle.md#serv-5e-tts-detection-从-adapter-metadata-的有序并集派生) | `ming_tts` 消歧、请求校验和 prompt delegation | ## 配置与 checkpoint 差异 From 46740127ba3838540362eddb00eae36f0545a70d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 00:34:39 +0800 Subject: [PATCH 116/465] knowledge(vllm-omni): create owner pages for ten models added after v0.26.0 The sync campaign halts when an upstream model directory has no knowledge owner, because filing a rule under a guessed owner gives it a stable ID that is never renumbered. Thirteen such names appear in the pinned range, the first at unit 115, so they are resolved here in one reviewed change rather than as thirteen halts. Ten are genuinely new models, each with its own upstream directory, and get an owner page: nemotron-voicechat dots-tts gepard minimax-music3 (AR / serving) lingbot-world pi0 sana-wm hidream-o1-image sana-video boogu-image (diffusion) The pages register ownership and the verified source entrypoints and nothing else. No behaviour, constants or gates are asserted: those are what the per-commit sync is for, and inventing them here would put unsourced claims into always-on briefing. models/_index.md carries , so new model directories need no index row. The remaining three are NPU platform files belonging to models we already own, and are declared in those pages sources rather than given pages of their own: minicpmo_4_5_code2wav -> minicpm-o-4-5 step_audio2_token2wav -> step-audio2 cosyvoice2_dit_attn -> step-audio2 The last one is not obvious from its name: it is an NPU attention patch for the CosyVoice2/Token2Wav DiT, and step_audio2_token2wav.py is what imports it. There is no upstream cosyvoice2 model directory, so it has no owner of its own. This follows the precedent the tree already sets -- qwen3-tts declares the NPU flat file qwen3_tts_tokenizer_v2.py, and higgs-audio declares both higgs_audio_v2/ and higgs_audio_v3/ -- which is also how the campaign now derives ownership instead of guessing from name similarity. No unit is consumed and audited_sha is untouched, so the pin chain is unaffected. Gates: check_knowledge_tree 0 errors / 12 pre-existing reminders, check_wiki_lint 0 errors across 325 pages, release audit CLEAN, test_knowledge_source 12 passed. Each page carries the four elements page-rules.md requires of a model owner entry: formal name and aliases, source paths, shared-module dependencies, and checkpoint/size/quantization. Every one is extracted from upstream at the pinned tip -- registry and pipeline_registry keys, imports crossing the package boundary, and deploy YAML headers. Where upstream declares nothing, the page says so rather than inventing it: hidream-o1-image and sana-video have neither a deploy preset nor a module docstring. Aliases are resolved through the constant a registry imports, not by matching the directory name. nemotron_labs_voicechat maps to NEMOTRON_VOICECHAT_PIPELINE and shares no substring with the module, yet it is the alias that makes bare vllm-omni serve work on a checkpoint whose config.json has no model_type. models/catalog.md is deliberately not updated here. It is a registry snapshot at audited_sha, which is still unit 113; these models enter the registry at their own units, and their sync commits will record them then. Co-Authored-By: Claude Opus 5 (1M context) --- .../vllm-omni/models/boogu-image/_index.md | 53 ++++++++++++++++++ .../repos/vllm-omni/models/dots-tts/_index.md | 50 +++++++++++++++++ .../repos/vllm-omni/models/gepard/_index.md | 50 +++++++++++++++++ .../models/hidream-o1-image/_index.md | 53 ++++++++++++++++++ .../vllm-omni/models/lingbot-world/_index.md | 53 ++++++++++++++++++ .../vllm-omni/models/minicpm-o-4-5/_index.md | 4 +- .../vllm-omni/models/minimax-music3/_index.md | 53 ++++++++++++++++++ .../models/nemotron-voicechat/_index.md | 56 +++++++++++++++++++ .../repos/vllm-omni/models/pi0/_index.md | 49 ++++++++++++++++ .../vllm-omni/models/sana-video/_index.md | 53 ++++++++++++++++++ .../repos/vllm-omni/models/sana-wm/_index.md | 53 ++++++++++++++++++ .../vllm-omni/models/step-audio2/_index.md | 4 +- 12 files changed, 527 insertions(+), 4 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/boogu-image/_index.md create mode 100644 knowledge/repos/vllm-omni/models/dots-tts/_index.md create mode 100644 knowledge/repos/vllm-omni/models/gepard/_index.md create mode 100644 knowledge/repos/vllm-omni/models/hidream-o1-image/_index.md create mode 100644 knowledge/repos/vllm-omni/models/lingbot-world/_index.md create mode 100644 knowledge/repos/vllm-omni/models/minimax-music3/_index.md create mode 100644 knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md create mode 100644 knowledge/repos/vllm-omni/models/pi0/_index.md create mode 100644 knowledge/repos/vllm-omni/models/sana-video/_index.md create mode 100644 knowledge/repos/vllm-omni/models/sana-wm/_index.md diff --git a/knowledge/repos/vllm-omni/models/boogu-image/_index.md b/knowledge/repos/vllm-omni/models/boogu-image/_index.md new file mode 100644 index 000000000..a48572d15 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/boogu-image/_index.md @@ -0,0 +1,53 @@ +--- +title: "Boogu Image" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/boogu_image/] +confidence: high +--- + +# Boogu Image + +## 正式名称与别名 + +- 知识树 owner：`models/boogu-image`；上游目录 `boogu_image`。 +- registry 登记的 architecture / stage key：`BooguImagePipeline`。 + +## 源码路径 + +共 5 个文件： + +- `vllm_omni/diffusion/models/boogu_image/` + +## 依赖的共享代码模块 + +- `vllm_omni/diffusion/data/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/attention/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/distributed/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/model_loader/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/models/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/platforms/` → 无对应 component owner +- `vllm_omni/diffusion/request/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/worker/` → [diffusion](../../components/diffusion/_index.md) + +## checkpoint、尺寸与量化 + +- 上游没有 deploy 预设。pipeline 模块自述：Native vLLM-Omni pipeline for Boogu-Image-0.1. Ported from the upstream ``boogu`` package (``boogu/pipelines/boogu/pipeline_boogu.py``) with the following changes: - Diffusers … + +## 什么时候查这里 + +只查 Boogu Image 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/dots-tts/_index.md b/knowledge/repos/vllm-omni/models/dots-tts/_index.md new file mode 100644 index 000000000..cb0f061d7 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/dots-tts/_index.md @@ -0,0 +1,50 @@ +--- +title: "dots.tts" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, serving] +sources: [vllm_omni/model_executor/models/dots_tts/, vllm_omni/deploy/dots_tts.yaml, vllm_omni/transformers_utils/configs/dots_tts.py] +confidence: high +--- + +# dots.tts + +## 正式名称与别名 + +- 知识树 owner：`models/dots-tts`；上游目录 `dots_tts`。 +- pipeline registry key：`dots_tts`（`vllm_omni/config/pipeline_registry.py`）。 +- registry 登记的 architecture / stage key：`dots_tts_talker`。 + +## 源码路径 + +共 8 个文件： + +- `vllm_omni/model_executor/models/dots_tts/` +- `vllm_omni/deploy/dots_tts.yaml` +- `vllm_omni/transformers_utils/configs/dots_tts.py` + +## 依赖的共享代码模块 + +- `vllm_omni/model_executor/models/` → [model-executor](../../components/model-executor/_index.md) +- `vllm_omni/config/stage_config/` → [configuration](../../components/configuration/_index.md) + +## checkpoint、尺寸与量化 + +- `dots_tts.yaml`：dots.tts deployment config (single-stage AR talker). Conservative first-integration settings: eager execution, no CUDA Graphs, no batched side-path optimizations yet — see voxcpm2.yaml for 配置：`async_chunk: false`；`dtype: bfloat16`。 + +## 什么时候查这里 + +只查 dots.tts 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/gepard/_index.md b/knowledge/repos/vllm-omni/models/gepard/_index.md new file mode 100644 index 000000000..64a2e4c07 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/gepard/_index.md @@ -0,0 +1,50 @@ +--- +title: "Gepard" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, serving] +sources: [vllm_omni/model_executor/models/gepard/, vllm_omni/deploy/gepard.yaml] +confidence: high +--- + +# Gepard + +## 正式名称与别名 + +- 知识树 owner：`models/gepard`；上游目录 `gepard`。 +- pipeline registry key：`gepard`（`vllm_omni/config/pipeline_registry.py`）。 +- registry 登记的 architecture / stage key：`gepard_talker`。 + +## 源码路径 + +共 6 个文件： + +- `vllm_omni/model_executor/models/gepard/` +- `vllm_omni/deploy/gepard.yaml` + +## 依赖的共享代码模块 + +- `vllm_omni/model_executor/models/` → [model-executor](../../components/model-executor/_index.md) +- `vllm_omni/platforms/` → 无对应 component owner +- `vllm_omni/config/stage_config/` → [configuration](../../components/configuration/_index.md) + +## checkpoint、尺寸与量化 + +- `gepard.yaml`：Gepard-1.0: single-stage native-AR FSQ/NanoCodec TTS (~0.5B). Target 1x L4 24GB. Zero-shot only; enforce_eager until the perf PR. 配置：`pipeline: gepard`；`async_chunk: false`；`dtype: bfloat16`。 + +## 什么时候查这里 + +只查 Gepard 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/hidream-o1-image/_index.md b/knowledge/repos/vllm-omni/models/hidream-o1-image/_index.md new file mode 100644 index 000000000..3d0dd58e2 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/hidream-o1-image/_index.md @@ -0,0 +1,53 @@ +--- +title: "HiDream-O1 Image" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/hidream_o1_image/] +confidence: high +--- + +# HiDream-O1 Image + +## 正式名称与别名 + +- 知识树 owner：`models/hidream-o1-image`；上游目录 `hidream_o1_image`。 +- registry 登记的 architecture / stage key：`HiDreamO1ImagePipeline`。 + +## 源码路径 + +共 3 个文件： + +- `vllm_omni/diffusion/models/hidream_o1_image/` + +## 依赖的共享代码模块 + +- `vllm_omni/diffusion/cache/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/attention/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/models/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/data/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/distributed/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/model_loader/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/profiler/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/worker/` → [diffusion](../../components/diffusion/_index.md) + +## checkpoint、尺寸与量化 + +- 上游没有 deploy 预设。模块也没有 docstring，因此 checkpoint、尺寸和量化差异在上游尚无声明，不在此处臆测；命中该 owner 的同步单元应补齐。 + +## 什么时候查这里 + +只查 HiDream-O1 Image 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/lingbot-world/_index.md b/knowledge/repos/vllm-omni/models/lingbot-world/_index.md new file mode 100644 index 000000000..3b3c62fa7 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/lingbot-world/_index.md @@ -0,0 +1,53 @@ +--- +title: "LingBot World" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/lingbot_world/] +confidence: high +--- + +# LingBot World + +## 正式名称与别名 + +- 知识树 owner：`models/lingbot-world`；上游目录 `lingbot_world`。 +- registry 登记的 architecture / stage key：`LingBotWorldCausalDMDPipeline`。 + +## 源码路径 + +共 5 个文件： + +- `vllm_omni/diffusion/models/lingbot_world/` + +## 依赖的共享代码模块 + +- `vllm_omni/experimental/ar_diffusion/` → 无对应 component owner +- `vllm_omni/diffusion/models/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/distributed/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/model_loader/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/layers/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/data/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/forward_context/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/profiler/` → [diffusion](../../components/diffusion/_index.md) + +## checkpoint、尺寸与量化 + +- 上游没有 deploy 预设。pipeline 模块自述：Request-scoped LingBot-World v2 causal DMD pipeline. + +## 什么时候查这里 + +只查 LingBot World 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 4360eda97..3ba8209e5 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -1,10 +1,10 @@ --- title: "MiniCPM-o 4.5" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-04 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py] +sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/minimax-music3/_index.md b/knowledge/repos/vllm-omni/models/minimax-music3/_index.md new file mode 100644 index 000000000..7b059e4a7 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-music3/_index.md @@ -0,0 +1,53 @@ +--- +title: "MiniMax Music3" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, serving] +sources: [vllm_omni/model_executor/models/minimax_music3/, vllm_omni/deploy/minimax_music3.yaml, vllm_omni/deploy/minimax_music3_2gpu.yaml, vllm_omni/entrypoints/openai/tts_adapters/minimax_music3.py, vllm_omni/model_executor/stage_input_processors/minimax_music3.py, vllm_omni/transformers_utils/configs/minimax_music3.py] +confidence: high +--- + +# MiniMax Music3 + +## 正式名称与别名 + +- 知识树 owner：`models/minimax-music3`；上游目录 `minimax_music3`。 +- pipeline registry key：`minimax_music3`（`vllm_omni/config/pipeline_registry.py`）。 + +## 源码路径 + +共 15 个文件： + +- `vllm_omni/model_executor/models/minimax_music3/` +- `vllm_omni/deploy/minimax_music3.yaml` +- `vllm_omni/deploy/minimax_music3_2gpu.yaml` +- `vllm_omni/entrypoints/openai/tts_adapters/minimax_music3.py` +- `vllm_omni/model_executor/stage_input_processors/minimax_music3.py` +- `vllm_omni/transformers_utils/configs/minimax_music3.py` + +## 依赖的共享代码模块 + +- `vllm_omni/model_executor/models/` → [model-executor](../../components/model-executor/_index.md) +- `vllm_omni/config/stage_config/` → [configuration](../../components/configuration/_index.md) + +## checkpoint、尺寸与量化 + +- `minimax_music3.yaml`：MiniMax-Music3 deploy: Stage 0 (AR talker) -> Stage 1 (acoustic decoder), both on one 140 GB card, streaming 200-frame windows over shared memory. 配置：`pipeline: minimax_music3`；`async_chunk: true`。 +- `minimax_music3_2gpu.yaml`：MiniMax-Music3 two-GPU layout: AR talker on device 0, acoustic decoder on device 1. Splitting the stages removes the memory-budget compromise of the single-card 配置：`pipeline: minimax_music3`；`async_chunk: true`。 + +## 什么时候查这里 + +只查 MiniMax Music3 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md b/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md new file mode 100644 index 000000000..f655b09b0 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md @@ -0,0 +1,56 @@ +--- +title: "NVIDIA Nemotron-Labs VoiceChat" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, serving] +sources: [vllm_omni/deploy/nemotron_labs_voicechat.yaml, vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/asr/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py] +confidence: high +--- + +# NVIDIA Nemotron-Labs VoiceChat + +## 正式名称与别名 + +- 知识树 owner：`models/nemotron-voicechat`；上游目录 `nemotron_voicechat`。 +- pipeline registry key：`nemotron_labs_voicechat`；`nemotron_voicechat`（`vllm_omni/config/pipeline_registry.py`）。 +- registry 登记的 architecture / stage key：`nemotron_voicechat_code2wav`；`nemotron_voicechat_talker`；`nemotron_voicechat_thinker`。 + +## 源码路径 + +共 30 个文件： + +- `vllm_omni/deploy/nemotron_labs_voicechat.yaml` +- `vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml` +- `vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml` +- `vllm_omni/experimental/fullduplex/nemotron_voicechat/` +- `vllm_omni/model_executor/models/nemotron_voicechat/` +- `vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/` +- `vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/asr/` +- `vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py` + +## 依赖的共享代码模块 + +- `vllm_omni/experimental/fullduplex/` → 无对应 component owner +- `vllm_omni/model_executor/models/` → [model-executor](../../components/model-executor/_index.md) +- `vllm_omni/config/stage_config/` → [configuration](../../components/configuration/_index.md) + +## checkpoint、尺寸与量化 + +- `nemotron_labs_voicechat.yaml`：NemotronVoiceChat-11B offline speech-to-speech deployment. 3 stages on one GPU: thinker (NemotronH AR) -> talker (EAR-TTS AR) -> code2wav (RVQ-VAE decode). + +## 什么时候查这里 + +只查 NVIDIA Nemotron-Labs VoiceChat 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/pi0/_index.md b/knowledge/repos/vllm-omni/models/pi0/_index.md new file mode 100644 index 000000000..582c40770 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/pi0/_index.md @@ -0,0 +1,49 @@ +--- +title: "Pi0" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/pi0/, vllm_omni/deploy/pi0.yaml] +confidence: high +--- + +# Pi0 + +## 正式名称与别名 + +- 知识树 owner：`models/pi0`；上游目录 `pi0`。 +- pipeline registry key：`pi0`（`vllm_omni/config/pipeline_registry.py`）。 +- registry 登记的 architecture / stage key：`Pi0Pipeline`。 + +## 源码路径 + +共 5 个文件： + +- `vllm_omni/diffusion/models/pi0/` +- `vllm_omni/deploy/pi0.yaml` + +## 依赖的共享代码模块 + +- `vllm_omni/diffusion/data/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/request/` → [diffusion](../../components/diffusion/_index.md) + +## checkpoint、尺寸与量化 + +- `pi0.yaml`：π0 (Pi-Zero) VLA deploy: single diffusion stage. Topology (PI0_PIPELINE) is declared in vllm_omni/diffusion/models/pi0/pipeline_pi0.py. 配置：`pipeline: pi0`；`async_chunk: false`；`dtype: float32`。 + +## 什么时候查这里 + +只查 Pi0 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/sana-video/_index.md b/knowledge/repos/vllm-omni/models/sana-video/_index.md new file mode 100644 index 000000000..7f7dc9845 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/sana-video/_index.md @@ -0,0 +1,53 @@ +--- +title: "SANA Video" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/sana_video/] +confidence: high +--- + +# SANA Video + +## 正式名称与别名 + +- 知识树 owner：`models/sana-video`；上游目录 `sana_video`。 +- registry 登记的 architecture / stage key：`SanaImageToVideoPipeline`；`SanaVideoPipeline`。 + +## 源码路径 + +共 5 个文件： + +- `vllm_omni/diffusion/models/sana_video/` + +## 依赖的共享代码模块 + +- `vllm_omni/diffusion/distributed/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/attention/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/models/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/data/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/offloader/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/request/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/worker/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/cache/` → [diffusion](../../components/diffusion/_index.md) + +## checkpoint、尺寸与量化 + +- 上游没有 deploy 预设。模块也没有 docstring，因此 checkpoint、尺寸和量化差异在上游尚无声明，不在此处臆测；命中该 owner 的同步单元应补齐。 + +## 什么时候查这里 + +只查 SANA Video 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/sana-wm/_index.md b/knowledge/repos/vllm-omni/models/sana-wm/_index.md new file mode 100644 index 000000000..e420803c4 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/sana-wm/_index.md @@ -0,0 +1,53 @@ +--- +title: "SANA-WM" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, diffusion] +sources: [vllm_omni/diffusion/models/sana_wm/] +confidence: high +--- + +# SANA-WM + +## 正式名称与别名 + +- 知识树 owner：`models/sana-wm`；上游目录 `sana_wm`。 +- registry 登记的 architecture / stage key：`SanaWmPipeline`。 + +## 源码路径 + +共 7 个文件： + +- `vllm_omni/diffusion/models/sana_wm/` + +## 依赖的共享代码模块 + +- `vllm_omni/diffusion/models/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/layers/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/distributed/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/attention/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/data/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/model_loader/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/offloader/` → [diffusion](../../components/diffusion/_index.md) +- `vllm_omni/diffusion/profiler/` → [diffusion](../../components/diffusion/_index.md) + +## checkpoint、尺寸与量化 + +- 上游没有 deploy 预设。pipeline 模块自述：Sana-WM pipeline integration. This module wires the registry-visible surface, release-layout validation, and the Stage-1 sampling loop. + +## 什么时候查这里 + +只查 SANA-WM 专有的行为、常量、注册入口和验证合同。 + +## 不放什么 + +上面列出的共享模块的执行、调度、加载或 serving 合同属于 +[components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 +清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | diff --git a/knowledge/repos/vllm-omni/models/step-audio2/_index.md b/knowledge/repos/vllm-omni/models/step-audio2/_index.md index a968fd9fa..8f1a7d358 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/_index.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/_index.md @@ -1,10 +1,10 @@ --- title: "Step-Audio2（音频 token 内嵌 LM 词表的语音对话）" created: 2026-07-21 -updated: 2026-09-02 +updated: 2026-09-04 type: index tags: [vllm-omni, models] -sources: ["PR #5067", "PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py] +sources: ["PR #5067", "PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py, vllm_omni/platforms/npu/models/step_audio2_token2wav.py, vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py] --- # Step-Audio2 From 28c758e7659c099f544a7227f4dea9707cca0201 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 00:41:20 +0800 Subject: [PATCH 117/465] knowledge(vllm-omni): sync 9b0df390 (PR #4795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 9b0df3909ae06207337cff817ce47e09cbbfe697 Upstream-PR: #4795 Knowledge-Delta: 新增 forced-aligner pooling stage 的配置、跨 stage bridge、scheduler 解码失败和 TTS transport 审查合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/configuration/rules.md | 13 +++++++++++-- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/model-executor/rules.md | 12 ++++++++++-- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/rules.md | 12 ++++++++++-- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/components/serving/rules.md | 12 ++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 9 ++++----- 13 files changed, 54 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ce92afd75..f4f5654f8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3c3989ed04300d396b3f6b44e9c0b70a0f9cabe0 + previous_audited_sha: a15b5b4bcda33a9fbd24017381d1f9423d1e840d audited_ref: main - audited_sha: a15b5b4bcda33a9fbd24017381d1f9423d1e840d + audited_sha: 9b0df3909ae06207337cff817ce47e09cbbfe697 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a1671e7ba..900b4344e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -a15b5b4bcda33a9fbd24017381d1f9423d1e840d`. The canonical machine baseline +9b0df3909ae06207337cff817ce47e09cbbfe697`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7b919d802..1eccadca8 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a15b5b4b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9b0df390` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 78fbd18a2..8b1eb51e8 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 配置开发门禁" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795"] --- # vLLM-Omni 配置开发门禁 @@ -179,3 +179,12 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR 混入 config migration。 - 验收：两种 wiring 都从同一 topology 展开，配置 diff 只包含预期 deploy 字段； 同卡多 stage 的 `gpu_memory_utilization` 总预算不超过可用容量。 + +### CONF-5c — 可选辅助 pooling stage 必须在最终 topology 中显式注入 + +- 触发：为现有 pipeline 增加可选辅助 stage，或新增 `--forced-aligner`、`--forced-aligner-config`、`--forced-aligner-device` 这类只由 orchestrator 读取的开关。 +- 强制：在最终 merge 前把 CLI/YAML 解析为一个 canonical stage config；任一 aligner 来源生效时复制 pipeline/deploy，追加连接到前一 stage 的 `runner="pooling"` 终端 stage，并将 `default_pooling_params.task`、设备和 decoder hook 投影到该 stage；无来源时保持原 topology。 +- 禁止：只检查 `forced_aligner` 而忽略 config-only 注入；原地修改调用方的 pipeline/deploy；把 orchestrator-only flags 泄漏为每个 stage 的 engine 参数；用 `execution_type` 猜测应构造 `PoolingParams` 还是生成参数。 +- 验收：覆盖模型参数、config-only、无参数三态；断言原 pipeline/deploy 未改变，最终逐 stage config 含正确模型、`runner`、pooling task、设备和 decoder hook，并证明 stage 初始化按 `runner` 选择 `PoolingParams`。 +^[PR #4795] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 66d34d53e..444d104e5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a15b5b4b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9b0df390` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 29ce110f0..f99d28b19 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a15b5b4b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9b0df390` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index be6f1d948..63b67fb44 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a15b5b4b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9b0df390` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 77df55736..c150ffaa4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -1,10 +1,10 @@ --- title: "Model Executor 规则" created: 2026-07-10 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795"] --- # Model Executor 规则 @@ -259,6 +259,14 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 验收：覆盖 source 被下一 step 覆写、ragged list shape/value/alias、普通路径与 finish/abort 清理。^[PR #5957] +### EXEC-1h — pooling stage 的输入输出必须走显式跨 stage bridge + +- 触发：新增非生成 pooling stage，或在 batch pipeline 中跨 stage 传递音频、文本、multimodal payload 与解码结果。 +- 强制：producer 只在上游请求完成后按原 request index 构造输入，分词一次并随音频携带 `aligner_words`、language 和真实 duration；pooling runner 识别 `is_pooling_model`，绕过仅适用于 AR hidden state 的 Omni prefix-cache/mm 提取，decoder 输出固定的 int32 `[n_words, 2]`；最终 consumer 使用同一份 words、duration 和原 batch 映射。 +- 禁止：把 pooling 请求当作 `generate` 或隐式广播单请求 metadata；按 downstream position 压缩原 batch index；重新分词后与 interval 做未验证的 positional zip；把空结果、解码失败或缺失 word metadata 伪装成 `[0, 0]` 的有效对齐；假设真实 `OmniRequestOutput` 暴露测试 fake 才有的 `additional_information`。 +- 验收：用真实 stage wrapper、`PoolingOutput` 和 `OmniRequestOutput` 做 mixed-batch 测试，分别断言音频完成门槛、metadata/shape、duration clamp、原 request 标签和 mismatch→无时间戳；同时验证普通生成 stage 的输入输出不受影响。 +^[PR #4795] + ### EXEC-6a — shared image example 先建 canonical envelope，model-extra 只做特化变换 - 触发：修改 shared T2I/I2V example、`model_extras` prompt registry 或模型 prompt builder。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bea772b13..a46706bfb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a15b5b4b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9b0df390` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 3efe70e62..96a60f60d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -1,10 +1,10 @@ --- title: "Scheduler 规则" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795"] --- # Scheduler 规则 @@ -223,6 +223,14 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 验收：覆盖完整 pair、不同进度、missing/split、parent abort 和 companion abort， 断言请求不会挂死、错误归属保持 request-local、队列和 connector state 都释放。 +## SCHED-5e — pooling output decoder 必须覆盖两个 scheduler 且失败即为 request error + +- 触发：pooling stage 配置 per-stage output decoder，或修改 `OmniARScheduler`、`OmniGenerationScheduler` 的 pooling output 完成路径。 +- 强制：两个 scheduler 都经共享 mixin 在 terminal status 处理前调用 decoder；成功时把解码 payload 写入 `EngineCoreOutput`，失败时只将当前 request 置为 `FINISHED_ERROR`、保留错误原因并停止继续调度。 +- 禁止：只在 AR scheduler 接 hook；decoder 异常 fail-open 为空成功；先完成请求再解码；把原始 pooler tensor 或错误 request 的 payload 发送给下游。 +- 验收：分别覆盖 AR/generation scheduler 的成功、decoder 抛异常、无 decoder 和同批健康 request；断言成功 payload、`FinishReason.ERROR`、request-local 错误归属及无跨请求污染。 +^[PR #4795] + ## SCHED-6a — async discard 的计数单位必须与 stale drain 一致 - 触发：上游 scheduler 改动异步占位、stale output 或 streaming segment replacement。 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6febbbfee..1978486df 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a15b5b4b` 验证存在 +- 源码校验：以上路径均已在 `main @ 9b0df390` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 62dc19c77..842d5bea6 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -1,10 +1,10 @@ --- title: "Serving 规则" created: 2026-07-20 -updated: 2026-09-03 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795"] confidence: high --- @@ -111,6 +111,14 @@ confidence: high 1808.96→1235.26 ms、peak RSS 8.874→4.141 GiB、above-resident 5.393→0.660 GiB，两个 hashes 相同。PR body 的 362-frame 与 4×MI300X E2E 是另一组观察，不能混合或泛化。^[PR #5732] +## SERV-1f — TTS word timestamps 必须按 transport capability 显式门禁和传输 + +- 触发：公开 `word_timestamps`、forced-aligner CLI/config，或修改 HTTP non-stream、SSE 和 WebSocket speech transport。 +- 强制：服务启动时统一记录 aligner capability；无 aligner 时 `word_timestamps=true` 明确返回 400。HTTP non-stream 将短 JSON 时间戳放入 `X-Word-Timestamps`，超过 4096 bytes 用可检测的 `X-Word-Timestamps-Omitted`；WebSocket 仅在 `stream_audio=true` 且 PCM 合同满足时，在每句音频结束后发送 trailing timestamps frame。 +- 禁止：无 aligner 时静默返回 200 音频；让 `stream=true` 的 HTTP 路径伪装支持该字段；把长 header 静默丢弃；以 in-process sidecar 或离线示例替代 pipeline stage 的真实输出。 +- 验收：覆盖 non-stream、HTTP streaming、WebSocket、无/有 aligner、短/超限 alignment 和多句音频；分别断言 400、header/frame 内容、超限标记，以及音频在时间戳缺失或超限时仍按合同返回。 +^[PR #4795] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 780d52eb6..a332b319c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a15b5b4b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9b0df390`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,8 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `3c3989ed` 四个 inventory 均无变化；目标 revision 只修改 repository skill 的 -examples policy，不是 runtime registry 或支持范围变更。 +本次相对 `a15b5b4b` 本轮沉淀了可选 forced-aligner pooling stage 的配置注入、跨 stage 输入输出桥接、scheduler 解码失败处理和 TTS 时间戳传输合同。离线示例删除与新增测试/文档未单独沉淀。不能据此推断离线时间戳仍受支持、所有 pooling stage 都具备相同语义，或生产链路已经可靠传递 aligner words；当前实现仍需警惕 words 重分段及测试 fake payload 与真实 OmniRequestOutput 不一致。 ## AR/omni 模型族（29） @@ -89,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 3c3989ed \ - --to a15b5b4b \ + --from a15b5b4b \ + --to 9b0df390 \ --repo \ --mode report-only ``` From 2fd9df644922e8ae11c7ee5640bb19793cc3828f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 01:29:46 +0800 Subject: [PATCH 118/465] knowledge(vllm-omni): sync 5be7541d (PR #5842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 5be7541d01ccc4f34f8d2049cc9780ba67a7f26f Upstream-PR: #5842 Knowledge-Delta: 新增 Nemotron VoiceChat 的 pipeline alias、hybrid-Mamba cache dtype、累积跨 stage payload、stateful async-chunk 与 full-payload admission 审查规则。 --- adapters/vllm_omni/release_baseline.yaml | 16 ++++++++-------- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/rules.md | 18 +++++++++++++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 10 +++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 11 ++++++++++- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 12 files changed, 55 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f4f5654f8..212356618 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a15b5b4bcda33a9fbd24017381d1f9423d1e840d + previous_audited_sha: 9b0df3909ae06207337cff817ce47e09cbbfe697 audited_ref: main - audited_sha: 9b0df3909ae06207337cff817ce47e09cbbfe697 + audited_sha: 5be7541d01ccc4f34f8d2049cc9780ba67a7f26f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 81 - sha256: 6526182d39857c0d9e2e04c1fb4b69606aa8e93b3019f2b471425722a9fcc592 + count: 84 + sha256: 18b2cca9644d8f2054029272c2fc5e3aa3c30e8a63ccf653d5293c1f9312c2a9 diffusion: count: 60 sha256: ae57a7d6867352714c00b5b93a5f22874ebc2f370cd32067925bb14ec84214f8 pipelines: - count: 53 - sha256: 152a88ee3e6d7dbb048d77eba3787361ab3820d289a01d562b6a84f4c142a960 + count: 55 + sha256: 0fb5191131211a863c0f81991bfb24990682cea47d6064f8a781d25ef0eda858 deploy_yamls: - count: 81 - sha256: 21e1b8c138a221ede273639d1c05599954489740ebaeff31d0ea3a8b59af3e48 + count: 83 + sha256: 015a788739d95fb9c89223067f3116bd6913e4d38a2fb96d1a25f6bdb9a47e88 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 900b4344e..571ce6d2c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -9b0df3909ae06207337cff817ce47e09cbbfe697`. The canonical machine baseline +5be7541d01ccc4f34f8d2049cc9780ba67a7f26f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1eccadca8..d4057be8f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9b0df390` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5be7541d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 8b1eb51e8..3e00ff521 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842"] --- # vLLM-Omni 配置开发门禁 @@ -82,6 +82,14 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR 只有 `CONF-数字字母` 是本节可审计规则 ID。 +### VOMNI-CFG-1j — hybrid-Mamba stage 的 SSM cache dtype 必须贯穿 typed projection + +- 触发：hybrid-Mamba stage 新增 SSM cache dtype，或同一模型提供普通与 async-chunk deploy 配置。 +- 强制：把 `mamba_ssm_cache_dtype` 同时声明在 typed shared/stage config 与最终 deploy stage，并投影到 vLLM `CacheConfig`；模型 `dtype` 与 SSM state dtype 分开验收，Nemotron thinker 默认保持 `float32`。 +- 禁止：只在 YAML 注释或模型内部设置 cache dtype；用 stage 的整体 `dtype` 替代 SSM state dtype；只验证 dataclass 能构造而不验证最终 runtime consumer。 +- 验收：从 deploy/structured/legacy 入口读取非默认值，断言 sync 与 async thinker 的最终 stage config 和 vLLM cache consumer 均得到 `mamba_ssm_cache_dtype=float32`，并覆盖 bf16 模型、fp32 SSM state 的组合。 +^[PR #5842] + ### CONF-1a — 多 stage 共卡时 diffusion stage 必须显式设 gpu_memory_utilization - 触发：多 stage 模型 stage 共卡时，分布式测试在模型加载期 CUDA OOM。 @@ -188,3 +196,11 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 验收：覆盖模型参数、config-only、无参数三态；断言原 pipeline/deploy 未改变，最终逐 stage config 含正确模型、`runner`、pooling task、设备和 decoder hook，并证明 stage 初始化按 `runner` 选择 `PoolingParams`。 ^[PR #4795] +### CONF-5d — 无 `model_type` 的多 stage checkpoint 必须闭合别名与架构路由 + +- 触发：接入缺少 `model_type` 或 `architectures` 的 NeMo 风格多 stage checkpoint，或新增其 pipeline alias、architecture 路由与默认 deploy 配置。 +- 强制：同时登记所有 stage architecture 到 `_ARCH_TO_MODEL_TYPE`，注册统一 `NemotronVoiceChatConfig`，让 alias、path-basename fallback 与 `default_deploy_config_name` 指向同一 pipeline；不得只依赖路径猜测。 +- 禁止：只注册 thinker 或只提供一个 alias；让空的 `model_arch` 覆盖 checkpoint architecture；让 sync 与 async deploy 配置解析出不同的 stage 拓扑。 +- 验收：用 bare `config.json` 和三个 architecture 分别验证 model type、pipeline、stage config 与默认 deploy 名称；sync/async 配置只切换 processor wiring，并保持三 stage 顺序与 stage architecture 一致。 +^[PR #5842] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 444d104e5..67ada1a79 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9b0df390` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5be7541d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f99d28b19..03f860c35 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9b0df390` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5be7541d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 63b67fb44..a096fff31 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9b0df390` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5be7541d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index c150ffaa4..d7d53f28e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py"] --- # Model Executor 规则 @@ -267,6 +267,14 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 验收：用真实 stage wrapper、`PoolingOutput` 和 `OmniRequestOutput` 做 mixed-batch 测试，分别断言音频完成门槛、metadata/shape、duration clamp、原 request 标签和 mismatch→无时间戳；同时验证普通生成 stage 的输入输出不受影响。 ^[PR #4795] +### EXEC-1i — 有状态跨 stage AR bridge 必须按 request 累积并完整消费 payload + +- 触发：多 stage AR 模型在 `preprocess` 中按 request 保存 KV、音频帧或 TTS 状态，并通过 connector 传递累积 timeline/code payload。 +- 强制：以 request id 隔离 session；无法证明 batch-safe 时显式限制 `max_num_seqs=1` 并在 request 完成/abort 时清理；下游每次采用累积 payload，单次 wake drain 所有尚未消费的位置，prompt-region 状态也必须先推进；code2wav 只接受真实 `[frames, 31]` codec codes。 +- 禁止：按一个 wake 只执行一个新位置；把延迟或合并的 chunk 当作只含一个 token；缺失 connector payload 时把 placeholder `input_ids` 当 codec codes 解码；用静默截断或 clamp 掩盖错误 shape/range。 +- 验收：覆盖 delayed/coalesced chunks、首个 PAD prompt chunk、zero-progress wake、最终 marker、缺失 payload、flat/二维 code shape、越界 code 与 session cleanup；断言 talker 与 code2wav 的真实输出字段和音频类型。 +^[PR #5842] + ### EXEC-6a — shared image example 先建 canonical envelope，model-extra 只做特化变换 - 触发：修改 shared T2I/I2V example、`model_extras` prompt registry 或模型 prompt builder。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a46706bfb..0b9e3e69d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9b0df390` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5be7541d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 96a60f60d..06c222fc4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842"] --- # Scheduler 规则 @@ -255,3 +255,12 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 机制与边界见 [architecture](architecture.md)；跨 stage 数据面见 [Distributed 组件](../distributed/_index.md)。 + +## SCHED-6c — full-payload allowlist 必须精确绑定最终 consumer 与 async wiring + +- 触发：新增 async-chunk processor 或把某个 stage 加入 full-payload input allowlist。 +- 强制：allowlist 使用精确的 `(model_arch, stage_key)` consumer 配对；Nemotron 的 `talker -> code2wav` 才能接收完整 code payload，`thinker -> talker` 保持 token-only；sync full-payload builder 与 async-chunk builder 必须分别接入同一真实 code2wav consumer。 +- 禁止：按模型家族或 stage index 猜测 full-payload 能力；把 token-only 的 thinker→talker hop 放进 full-payload allowlist；让非 async deploy 逐 token搬运完整 code stack，或让 async 终端 chunk 缺少 finished/empty-terminal 语义。 +- 验收：覆盖 allowlisted code2wav 与 non-allowlisted talker 对照，分别验证普通 deploy 的一次 request-end 全 payload、streaming deploy 的累计 chunk/终端 chunk、connector requeue 和 abort cleanup。 +^[PR #5842] + diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1978486df..b519d10ef 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9b0df390` 验证存在 +- 源码校验：以上路径均已在 `main @ 5be7541d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a332b319c..489d102a2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9b0df390`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5be7541d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `a15b5b4b` 本轮沉淀了可选 forced-aligner pooling stage 的配置注入、跨 stage 输入输出桥接、scheduler 解码失败处理和 TTS 时间戳传输合同。离线示例删除与新增测试/文档未单独沉淀。不能据此推断离线时间戳仍受支持、所有 pooling stage 都具备相同语义，或生产链路已经可靠传递 aligner words；当前实现仍需警惕 words 重分段及测试 fake payload 与真实 OmniRequestOutput 不一致。 +本次相对 `9b0df390` 本轮新增 NVIDIA Nemotron-Labs VoiceChat 的三阶段 pipeline、hybrid-Mamba cache dtype、跨 stage 累积 payload 与 async-chunk 路由事实。不能据此推断已支持 full-duplex serving、batch>1 或 function-calling。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from a15b5b4b \ - --to 9b0df390 \ + --from 9b0df390 \ + --to 5be7541d \ --repo \ --mode report-only ``` From 589331498c0019d79228f448e4bc6331f96b7f85 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 01:31:34 +0800 Subject: [PATCH 119/465] knowledge(vllm-omni): sync 31384cb3 (PR #6048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 31384cb39cf107809e64338519de55e119361221 Upstream-PR: #6048 Knowledge-Delta: 本单元仅涉及 CI nightly 镜像发布与清理流程；当前路由 allowlist 没有可写的 CI 规则页，因此不沉淀知识规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 212356618..21483768c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9b0df3909ae06207337cff817ce47e09cbbfe697 + previous_audited_sha: 5be7541d01ccc4f34f8d2049cc9780ba67a7f26f audited_ref: main - audited_sha: 5be7541d01ccc4f34f8d2049cc9780ba67a7f26f + audited_sha: 31384cb39cf107809e64338519de55e119361221 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 571ce6d2c..7fe592845 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5be7541d01ccc4f34f8d2049cc9780ba67a7f26f`. The canonical machine baseline +31384cb39cf107809e64338519de55e119361221`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d4057be8f..752afb193 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5be7541d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 31384cb3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 67ada1a79..a77f8379c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5be7541d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 31384cb3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 03f860c35..5bee2fcda 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5be7541d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 31384cb3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a096fff31..a2b74dd45 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5be7541d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 31384cb3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0b9e3e69d..2238cd3ed 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5be7541d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 31384cb3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b519d10ef..6a7a70725 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5be7541d` 验证存在 +- 源码校验：以上路径均已在 `main @ 31384cb3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 489d102a2..82e19f6ce 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5be7541d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 31384cb3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `9b0df390` 本轮新增 NVIDIA Nemotron-Labs VoiceChat 的三阶段 pipeline、hybrid-Mamba cache dtype、跨 stage 累积 payload 与 async-chunk 路由事实。不能据此推断已支持 full-duplex serving、batch>1 或 function-calling。 +本次相对 `5be7541d` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 9b0df390 \ - --to 5be7541d \ + --from 5be7541d \ + --to 31384cb3 \ --repo \ --mode report-only ``` From 81c99f790717465c543d115697c331f3557674ae Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 01:47:41 +0800 Subject: [PATCH 120/465] knowledge(vllm-omni): sync 7cd826d2 (PR #6095) Upstream-Commit: 7cd826d28c7e986bbcdd740f099b4eb256b52c4f Upstream-PR: #6095 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 21483768c..2a8fb4e0b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5be7541d01ccc4f34f8d2049cc9780ba67a7f26f + previous_audited_sha: 31384cb39cf107809e64338519de55e119361221 audited_ref: main - audited_sha: 31384cb39cf107809e64338519de55e119361221 + audited_sha: 7cd826d28c7e986bbcdd740f099b4eb256b52c4f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7fe592845..21f4bcb54 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -31384cb39cf107809e64338519de55e119361221`. The canonical machine baseline +7cd826d28c7e986bbcdd740f099b4eb256b52c4f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 752afb193..7532703cc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 31384cb3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7cd826d2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a77f8379c..e66b243dc 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 31384cb3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 7cd826d2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5bee2fcda..ca237c9bf 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 31384cb3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7cd826d2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a2b74dd45..d253cb83a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 31384cb3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7cd826d2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2238cd3ed..affbb07ac 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 31384cb3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 7cd826d2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6a7a70725..bc8b9e756 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 31384cb3` 验证存在 +- 源码校验：以上路径均已在 `main @ 7cd826d2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 82e19f6ce..923bbc9f2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 31384cb3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7cd826d2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `5be7541d` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `31384cb3` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 5be7541d \ - --to 31384cb3 \ + --from 31384cb3 \ + --to 7cd826d2 \ --repo \ --mode report-only ``` From 5a40024b1878e34c2e7275fb52822aea32155426 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 01:49:16 +0800 Subject: [PATCH 121/465] knowledge(vllm-omni): sync 38a2500b (PR #5915) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 38a2500bdf887cd0d23c1c1616c59faec6ffd317 Upstream-PR: #5915 Knowledge-Delta: 为 MiniMax H3 现有 MMH3-1c 补录 PR #5915，确认 Qwen3-VL RMSNorm 共享实现与 NPU fused dispatch 合同已有规范覆盖。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 4 ++-- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2a8fb4e0b..23bda4e75 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 31384cb39cf107809e64338519de55e119361221 + previous_audited_sha: 7cd826d28c7e986bbcdd740f099b4eb256b52c4f audited_ref: main - audited_sha: 7cd826d28c7e986bbcdd740f099b4eb256b52c4f + audited_sha: 38a2500bdf887cd0d23c1c1616c59faec6ffd317 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 21f4bcb54..f4cfae3d9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -7cd826d28c7e986bbcdd740f099b4eb256b52c4f`. The canonical machine baseline +38a2500bdf887cd0d23c1c1616c59faec6ffd317`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7532703cc..312fbf7c7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7cd826d2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 38a2500b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e66b243dc..18b8375a8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7cd826d2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 38a2500b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ca237c9bf..b2147c74a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7cd826d2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 38a2500b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index d253cb83a..01d045cee 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7cd826d2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 38a2500b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index affbb07ac..0fb852bdd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7cd826d2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 38a2500b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index bc8b9e756..fced44a32 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7cd826d2` 验证存在 +- 源码校验：以上路径均已在 `main @ 38a2500b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 923bbc9f2..dcd967e08 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7cd826d2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 38a2500b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `31384cb3` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `7cd826d2` 本轮未新增规则条目或页面；将 PR #5915 补录为现有 MMH3-1c（Qwen3-VL 共享 RMSNorm、BF16 gamma、native FP32 累加与 NPU 分派）的一条证据。不得据此推断其他模型或平台已完成 fused RMSNorm 验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 31384cb3 \ - --to 7cd826d2 \ + --from 7cd826d2 \ + --to 38a2500b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index d1868f7b2..ab4a65e12 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -1,10 +1,10 @@ --- title: "MiniMax H3 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915"] confidence: high --- From 62896d81ef6ba844e6cc768eb76f62e75686d70e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 01:51:53 +0800 Subject: [PATCH 122/465] knowledge(vllm-omni): sync fcf10853 (PR #5146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: fcf1085395940df1f9486e2f1b6f11581d8002c9 Upstream-PR: #5146 Knowledge-Delta: #5146 将 OmniRequestOutput 从嵌套包装改为 RequestOutput 子类，并建立扁平 msgpack 编解码及旧嵌套 wire 格式迁移合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/distributed/rules.md | 11 +++++++++-- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/model-executor/rules.md | 9 ++++++++- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 30 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 23bda4e75..5b04cc124 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7cd826d28c7e986bbcdd740f099b4eb256b52c4f + previous_audited_sha: 38a2500bdf887cd0d23c1c1616c59faec6ffd317 audited_ref: main - audited_sha: 38a2500bdf887cd0d23c1c1616c59faec6ffd317 + audited_sha: fcf1085395940df1f9486e2f1b6f11581d8002c9 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f4cfae3d9..b89a973c9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -38a2500bdf887cd0d23c1c1616c59faec6ffd317`. The canonical machine baseline +fcf1085395940df1f9486e2f1b6f11581d8002c9`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 312fbf7c7..5c5c07da9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 38a2500b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fcf10853` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 18b8375a8..87f489a9b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 38a2500b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ fcf10853` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b2147c74a..2158e1a70 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 38a2500b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fcf10853` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 6158c507f..6b583fb2f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -1,10 +1,10 @@ --- title: "Distributed 传输规则" created: 2026-08-05 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py] +sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146"] confidence: high --- @@ -58,3 +58,10 @@ confidence: high 的旧 vLLM 无条件传参；把 TP/EP 配置存在当作 manager 已初始化。 - 验收：mock 新旧签名分别断言 EP 转发/旧版省略，普通 group 不启用；目标环境用真实 MoE DiT TP+EP smoke 验证 manager 非空。^[PR #5976] + +## DIST-3a — Omni 输出序列化必须按扁平 dataclass 合同兼容旧线格式 + +- 触发：修改 `OmniMsgpackEncoder`/`OmniMsgpackDecoder`、`OmniRequestOutput` 字段或跨 stage 输出 wire format。 +- 强制：当前 `OmniRequestOutput` 按其 dataclass 字段重建，不能落入基类 `RequestOutput` 的动态属性编码路径；decoder 只构造已知字段，并在旧 wire format 含嵌套 `request_output` 时完成一次向扁平对象的内容合并。基类 `RequestOutput` 的字段集合必须跟随 pinned upstream 合同。 +- 禁止：让 subclass 走基类序列化而丢失 diffusion/stage 字段或形成递归；不得把未知 key 直接传给 `RequestOutput`/dataclass 构造器；upstream 已移除的 `multi_modal_placeholders` 不得仅为旧动态属性重新作为当前字段恢复。 +- 验收：msgpack round-trip 分别覆盖 pipeline 文本输出和 diffusion 图像输出，保持 request metadata、prompt、token、outputs、finished、stage 字段及图像内容；旧嵌套 wire payload 能解码为扁平对象，编码不递归且解码后的字段可被直接消费。^[PR #5146] diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 01d045cee..f68227f88 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 38a2500b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fcf10853` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index d7d53f28e..1a0f854c2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146"] --- # Model Executor 规则 @@ -287,3 +287,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 验收：canonical builder 覆盖 omitted/empty/value negative prompt 与 PIL media identity；registry 覆盖 Bagel、MammothModa2、Ming、VACE custom path及 unknown identity path；Cosmos3/LingBot pipeline tests继续拥有模型专属 validation。^[PR #6049] + +### EXEC-7a — Omni 输出必须是扁平的 RequestOutput 子类 + +- 触发：修改 `OmniRequestOutput`、stage output wrapping、`omni.generate()` 返回对象或其 serving/offline 消费者。 +- 强制：`OmniRequestOutput` 必须继承 vLLM `RequestOutput` 并将继承字段作为 dataclass 字段直接声明；stage 原始输出必须通过 `from_stage_output()` 显式复制 `prompt`、`prompt_token_ids`、`outputs`、`finished` 等生成内容，源对象为 `OmniRequestOutput` 时再复制 diffusion 内容；消费者直接读取扁平字段，列表归一化只在真实列表边界完成。 +- 禁止：重新引入 `request_output` 嵌套字段、动态 pass-through property、修改 dataclass 生成的 `__init__` 或递归 `unwrap` 兼容层；不得在 serving、example 或工具代码中恢复 `output.request_output` 访问。 +- 验收：用真实 `RequestOutput` 和 `OmniRequestOutput` 分别验证 factory 复制的 prompt、token、outputs、finished 及 images/trajectory 内容；覆盖文本、音频、图像和 pipeline stage 的直接字段消费，并确认未完成 stage 的 `finished` 不被包装过程改写。^[PR #5146] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0fb852bdd..d1fea376d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 38a2500b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fcf10853` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fced44a32..785270b53 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 38a2500b` 验证存在 +- 源码校验：以上路径均已在 `main @ fcf10853` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index dcd967e08..1c2b65e63 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 38a2500b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fcf10853`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `7cd826d2` 本轮未新增规则条目或页面；将 PR #5915 补录为现有 MMH3-1c（Qwen3-VL 共享 RMSNorm、BF16 gamma、native FP32 累加与 NPU 分派）的一条证据。不得据此推断其他模型或平台已完成 fused RMSNorm 验证。 +本次相对 `38a2500b` 本轮新增 #5146 的统一输出对象合同：`OmniRequestOutput` 归入 model-executor，扁平对象的 msgpack 传输与旧嵌套线格式兼容归入 distributed。不要据此推断模型专有 modality 字段已经完成重构，也不要把示例、测试迁移或单次 round-trip 证据解释为所有模型和硬件的端到端 parity。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 7cd826d2 \ - --to 38a2500b \ + --from 38a2500b \ + --to fcf10853 \ --repo \ --mode report-only ``` From 1ca947aa3fba913d891cbe50bc5db8fc27bef220 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 02:41:31 +0800 Subject: [PATCH 123/465] knowledge(vllm-omni): split the bridge-batch group out of model-executor rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit 120 halted on the 32768-byte hard split gate, which is the halt the runbook anticipates: land a reviewed split as a non-unit commit, then resume. The driver rolled the unit back cleanly, so the page is 31496 B on disk and it was the post-apply size that crossed. The 跨 stage bridge 与 batch 合同 section had grown to 13298 B and stopped matching its own name, holding four review groups at once: bridge-batch, loader-contract, image-task-envelope and the newly opened EXEC-7. Extracting the group the section is actually named for keeps the split along a boundary that already exists in the 审查组 table rather than inventing one. EXEC-1a-1i (bridge-batch, 9 rules, 8474 B) -> rules-bridge-batch.md Every rule ID is unchanged, the group row in the parent 审查组 table points at the new page, and it is registered once in _index.md. Headroom after: rules.md 9711 B, the new page 23437 B. Four inbound anchors pointed at the moved rules -- from configuration/rules.md, indextts-2-5, and two qwen-omni pages -- and were retargeted. check_knowledge_tree validates file existence but not anchors, so these are invisible to it; the same class of break appeared in the earlier diffusion/serving split and is worth checking by hand after every extraction. No unit is consumed and audited_sha is untouched, so the pin chain is unaffected. Gates: tree 0 errors / 12 pre-existing reminders, wiki lint 0 errors across 326 pages, release audit CLEAN, test_knowledge_source 12 passed. Co-Authored-By: Claude Opus 5 (1M context) --- .../components/configuration/rules.md | 2 +- .../components/model-executor/_index.md | 4 +- .../model-executor/rules-bridge-batch.md | 112 ++++++++++++++++++ .../components/model-executor/rules.md | 97 +-------------- .../vllm-omni/models/indextts-2-5/rules.md | 4 +- .../vllm-omni/models/qwen-omni/_index.md | 4 +- .../models/qwen-omni/architecture.md | 4 +- 7 files changed, 123 insertions(+), 104 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 3e00ff521..be54209b8 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -76,7 +76,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR override 的 AR control，证明 input width 回退到该 stage text hidden size。Qwen2.5-Omni 当前为 Thinker=`thinker_config`、Talker=`talker_config`、Code2Wav=`thinker_config`，thinker-only 也用 `thinker_config`。最终 diff 只测 stage 名与 Qwen3 无 override control，未直接单测 Qwen2.5 - Talker overridden-width runner path；该缺口见 [EXEC-1d](../model-executor/rules.md#exec-1d-cross-stage-embedding-buffer-必须按-ingress-width-分配)。^[PR #5073] + Talker overridden-width runner path；该缺口见 [EXEC-1d](../model-executor/rules-bridge-batch.md#exec-1d-cross-stage-embedding-buffer-必须按-ingress-width-分配)。^[PR #5073] ## 部署配置与资源预算 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f68227f88..5efb4fdce 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -1,7 +1,7 @@ --- title: "Model Executor" created: 2026-07-10 -updated: 2026-08-23 +updated: 2026-09-04 type: index tags: [vllm-omni, components, model-executor] sources: [] @@ -35,3 +35,5 @@ sources: [] |---|---| | 理解共享职责和阶段边界 | [architecture](architecture.md) | | 根据 PR 描述直达 stage config、runner preprocess、stage runtime、bridge/batch 或 loader 的规则组与第一批源码 | [rules 与代码地图](rules.md) | + +| runtime info、跨 stage payload、batch 与 request RNG 合同 | [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | \ No newline at end of file diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md new file mode 100644 index 000000000..ad5afe002 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -0,0 +1,112 @@ +--- +title: "跨 stage bridge 与 batch 合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, model-executor] +sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py] +confidence: high +--- + +# 跨 stage bridge 与 batch 合同 + +`bridge-batch` 审查组的 `EXEC-1a`–`EXEC-1i`：runtime info、跨 stage payload、batch +与 request RNG。触发条件和其余审查组见 [model-executor 共享规则](rules.md) 的 +Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x`。 + +## EXEC-1a — 从 producer 字段追到下一 stage consumer 和最终输出包装 + +- 触发：新模型、多阶段 pipeline、stage wrapper、runtime info 或 multimodal payload。 +- 强制：逐段记录 runner 写入字段、传输后的字段名/shape、下一 stage 读取位置，以及最终 + `OmniOutput`/multimodal payload 的包装。loader 或模型 class 单独可调用不能代替真实 + stage handoff。 +- 禁止：让 tuple waveform/hidden state 依赖 runner 的隐式猜测；bridge key 不一致时用 + fallback 掩盖。 +- 验收：测试从真实 stage wrapper 输入开始，断言下一 stage 收到逐请求字段并得到公开 + 输出类型。MiniCPM-o 的具体合同见 + [MiniCPM-o 4.5 规则](../../models/minicpm-o-4-5/rules.md)。 ^[PR #3642] + +## EXEC-1b — stage 声明 batch 能力就必须逐请求消费 runtime info + +- 触发：`max_num_seqs > 1`、batch handoff 或 wrapper 接收 `runtime_info` 列表。 +- 强制：输出按请求索引与输入一一对应；无法安全逐请求处理时把并发上限显式收紧为 1。 +- 禁止：只消费 `runtime_info[0]`，或把单元素 waveform/metadata 广播给整个 batch。 +- 验收：至少两个不同输入的同批测试，分别断言 bridge、输出和错误归属；不能重复相同 + prompt 让串线不可见。partial downstream subset 必须保留原 `req_id_to_index`：跳过 + 中间请求后，后续请求的 tensor slice 和 list-valued payload 仍取原 batch index，不能 + 压缩到 downstream position。 ^[PR #3642] ^[PR #5310] + +## EXEC-1c — 请求随机状态跨 batching 和 yield 保持请求所有权 + +- 触发：AR/talker adapter 接收 seed/sampling knob，或修改 batch compaction/reorder、逐 token loop。 +- 强制：请求值在 adapter/model 构造前到达真实 sampling consumer；使用 request-local generator。 + 若依赖必须临时改 global RNG，只能在无 yield 的窄上下文 save/restore。 +- 禁止：deploy 默认覆盖请求 seed；共享 generator；每步复制完整历史或创建无界 + `batch*vocab` 临时量；global RNG 状态跨 yield 泄漏到兄弟请求。 +- 验收：同 seed 同输出、异 seed 不同输出，batch reorder/compaction 后逐请求结果稳定；全局 RNG + 前后相同，计数器证明目标分支实际消费请求参数。 ^[PR #3422] ^[PR #5074] ^[PR #5792] + +## EXEC-1d — cross-stage embedding buffer 必须按 ingress width 分配 + +- 触发：stage 的输入 embedding 在 model forward 或 preprocess 内投影，或修改 shared AR runner + 的 `inputs_embeds` buffer。 +- 强制：buffer second dimension 来自 `model_config.get_inputs_embeds_size()`；它表示进入 stage + 时的 pre-projection width，与内部 `hf_text_config.hidden_size` 分开。没有 stage override 时 + helper 必须回退到内部 hidden size，保持其他 AR stage 的既有行为。 +- 禁止：把内部 hidden size 当跨 stage wire shape；反过来用某模型的外部宽度扩大所有 stage； + 只测 config helper 而不覆盖发生宽度差异的正向 runner/model path。 +- 验收：至少一个 ingress≠internal 的正向 case 与一个无 override control，断言真实 buffer shape + 和 projection consumer。Qwen2.5 Talker 当前接收 3584-wide Thinker state，并在 forward 内投影 + 到 896；Qwen3 在 preprocessing 先投影，因此走 hidden-size fallback。PR #5073 的最终单测只有 + Qwen3 control，没有直接钉住 Qwen2.5 正向 buffer，后者仍是回归缺口。^[PR #5073] + +## EXEC-1e — upstream registry 重名时 Omni override 与 plain-vLLM forward 必须同时成立 + +- 触发：上游开始注册 Omni 同名 architecture，或 model runner/forward/capture/dummy-run 接口变化。 +- 强制：全局 registry 无条件重注册 Omni owner；非 staged 模式为 Qwen Omni 选择 thinker，并返回 + stock runner 可消费的 bare tensor；只有 staged talker consumer 存在时才 capture hidden layers。 + runner override 必须接受上游新增 kwargs；text/MoE dummy input 可按 upstream 要求 randomize，但 + code2wav 等结构化 codec id 只能接受参数而不得随机化。worker profiling 同步保存上游新增结果字段。 +- 禁止：因 upstream 已有同名 arch 就跳过 Omni 注册；plain serve 返回 staged tuple；把 vocab-uniform + id 喂给 codec codebook；用 `**kwargs` 隐藏未审查的签名漂移。 +- 验收：registry collision 解析到 Omni class；Qwen2.5/Qwen3 plain thinker 与 staged capture 分别断言 + output shape，PP intermediate 原样透传；dummy-run 覆盖 text randomize 与 generation no-randomize， + 并对 profiling result 做字段 parity。^[PR #5976] + +## EXEC-1f — multimodal wrapper 与 hash algorithm 必须通过 live context 传播 + +- 触发：upstream multimodal input 开始保留原始 bytes、修改 hash 签名或 processor item wrapper。 +- 强制：所有 `get_mm_hashes` / direct `MultiModalHasher.hash_kwargs` 调用传当前 multimodal config 的 + algorithm；processor 通过公开 `get(index)` 解包 `MediaWithBytes`，再把 frames/metadata 交给 HF。 +- 禁止：直接遍历 `.data` 绕过 unwrap；让 stage/replica UUID 前缀替代 content hash algorithm；只修 + 一个模型而不 census 自定义 processor 与 direct hasher caller。 +- 验收：不同 algorithm 进入 hash consumer；wrapped/unwrapped video 都生成同一 HF input，metadata + 保留；stage+replica scope 只包裹 base UUID 且用户 UUID 优先。^[PR #5976] + +## EXEC-1g — request-end payload 延迟 D2H 必须先取得 device snapshot + +- 触发：模型设置 `omni_payload_at_request_end`，或修改 full-payload accumulation、CUDA graph + 和 D2H 策略。 +- 强制：只在显式 opt-in、无 prefix cache 且下游消费完整 payload 时延迟 D2H；每步 + clone device tensor，相容 list 可 pack 后以 views 复原，结束才跨设备，中间 output 为 `None`。 +- 禁止：累计 graph/input-buffer alias，或扩散到普通逐步 payload。 +- 验收：覆盖 source 被下一 step 覆写、ragged list shape/value/alias、普通路径与 + finish/abort 清理。^[PR #5957] + +## EXEC-1h — pooling stage 的输入输出必须走显式跨 stage bridge + +- 触发：新增非生成 pooling stage，或在 batch pipeline 中跨 stage 传递音频、文本、multimodal payload 与解码结果。 +- 强制：producer 只在上游请求完成后按原 request index 构造输入，分词一次并随音频携带 `aligner_words`、language 和真实 duration；pooling runner 识别 `is_pooling_model`，绕过仅适用于 AR hidden state 的 Omni prefix-cache/mm 提取，decoder 输出固定的 int32 `[n_words, 2]`；最终 consumer 使用同一份 words、duration 和原 batch 映射。 +- 禁止：把 pooling 请求当作 `generate` 或隐式广播单请求 metadata；按 downstream position 压缩原 batch index；重新分词后与 interval 做未验证的 positional zip；把空结果、解码失败或缺失 word metadata 伪装成 `[0, 0]` 的有效对齐；假设真实 `OmniRequestOutput` 暴露测试 fake 才有的 `additional_information`。 +- 验收：用真实 stage wrapper、`PoolingOutput` 和 `OmniRequestOutput` 做 mixed-batch 测试，分别断言音频完成门槛、metadata/shape、duration clamp、原 request 标签和 mismatch→无时间戳；同时验证普通生成 stage 的输入输出不受影响。 +^[PR #4795] + +## EXEC-1i — 有状态跨 stage AR bridge 必须按 request 累积并完整消费 payload + +- 触发：多 stage AR 模型在 `preprocess` 中按 request 保存 KV、音频帧或 TTS 状态，并通过 connector 传递累积 timeline/code payload。 +- 强制：以 request id 隔离 session；无法证明 batch-safe 时显式限制 `max_num_seqs=1` 并在 request 完成/abort 时清理；下游每次采用累积 payload，单次 wake drain 所有尚未消费的位置，prompt-region 状态也必须先推进；code2wav 只接受真实 `[frames, 31]` codec codes。 +- 禁止：按一个 wake 只执行一个新位置；把延迟或合并的 chunk 当作只含一个 token；缺失 connector payload 时把 placeholder `input_ids` 当 codec codes 解码；用静默截断或 clamp 掩盖错误 shape/range。 +- 验收：覆盖 delayed/coalesced chunks、首个 PAD prompt chunk、zero-progress wake、最终 marker、缺失 payload、flat/二维 code shape、越界 code 与 session cleanup；断言 talker 与 code2wav 的真实输出字段和音频类型。 +^[PR #5842] + +相关执行流见 [model-executor architecture](architecture.md)；scheduler 侧的对应合同见 [scheduler 规则](../scheduler/rules.md)。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 1a0f854c2..b54f67ef4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -31,7 +31,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru |---|---|---| | `core` | 每次 model-executor 审查 | `EXEC-1a` | | `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | -| `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`, `EXEC-1b`, `EXEC-1c`, `EXEC-1d` | +| `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`–`EXEC-1i`，见 [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | | `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`, `EXEC-2b` | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a` | | `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | @@ -148,52 +148,6 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 ## 跨 stage bridge 与 batch 合同 -### EXEC-1a — 从 producer 字段追到下一 stage consumer 和最终输出包装 - -- 触发：新模型、多阶段 pipeline、stage wrapper、runtime info 或 multimodal payload。 -- 强制：逐段记录 runner 写入字段、传输后的字段名/shape、下一 stage 读取位置，以及最终 - `OmniOutput`/multimodal payload 的包装。loader 或模型 class 单独可调用不能代替真实 - stage handoff。 -- 禁止：让 tuple waveform/hidden state 依赖 runner 的隐式猜测；bridge key 不一致时用 - fallback 掩盖。 -- 验收：测试从真实 stage wrapper 输入开始，断言下一 stage 收到逐请求字段并得到公开 - 输出类型。MiniCPM-o 的具体合同见 - [MiniCPM-o 4.5 规则](../../models/minicpm-o-4-5/rules.md)。 ^[PR #3642] - -### EXEC-1b — stage 声明 batch 能力就必须逐请求消费 runtime info - -- 触发：`max_num_seqs > 1`、batch handoff 或 wrapper 接收 `runtime_info` 列表。 -- 强制：输出按请求索引与输入一一对应；无法安全逐请求处理时把并发上限显式收紧为 1。 -- 禁止：只消费 `runtime_info[0]`，或把单元素 waveform/metadata 广播给整个 batch。 -- 验收：至少两个不同输入的同批测试，分别断言 bridge、输出和错误归属；不能重复相同 - prompt 让串线不可见。partial downstream subset 必须保留原 `req_id_to_index`：跳过 - 中间请求后，后续请求的 tensor slice 和 list-valued payload 仍取原 batch index，不能 - 压缩到 downstream position。 ^[PR #3642] ^[PR #5310] - -### EXEC-1c — 请求随机状态跨 batching 和 yield 保持请求所有权 - -- 触发：AR/talker adapter 接收 seed/sampling knob，或修改 batch compaction/reorder、逐 token loop。 -- 强制：请求值在 adapter/model 构造前到达真实 sampling consumer；使用 request-local generator。 - 若依赖必须临时改 global RNG，只能在无 yield 的窄上下文 save/restore。 -- 禁止：deploy 默认覆盖请求 seed；共享 generator；每步复制完整历史或创建无界 - `batch*vocab` 临时量；global RNG 状态跨 yield 泄漏到兄弟请求。 -- 验收：同 seed 同输出、异 seed 不同输出，batch reorder/compaction 后逐请求结果稳定；全局 RNG - 前后相同，计数器证明目标分支实际消费请求参数。 ^[PR #3422] ^[PR #5074] ^[PR #5792] - -### EXEC-1d — cross-stage embedding buffer 必须按 ingress width 分配 - -- 触发：stage 的输入 embedding 在 model forward 或 preprocess 内投影，或修改 shared AR runner - 的 `inputs_embeds` buffer。 -- 强制：buffer second dimension 来自 `model_config.get_inputs_embeds_size()`；它表示进入 stage - 时的 pre-projection width，与内部 `hf_text_config.hidden_size` 分开。没有 stage override 时 - helper 必须回退到内部 hidden size，保持其他 AR stage 的既有行为。 -- 禁止：把内部 hidden size 当跨 stage wire shape；反过来用某模型的外部宽度扩大所有 stage； - 只测 config helper 而不覆盖发生宽度差异的正向 runner/model path。 -- 验收：至少一个 ingress≠internal 的正向 case 与一个无 override control，断言真实 buffer shape - 和 projection consumer。Qwen2.5 Talker 当前接收 3584-wide Thinker state，并在 forward 内投影 - 到 896；Qwen3 在 preprocessing 先投影，因此走 hidden-size fallback。PR #5073 的最终单测只有 - Qwen3 control，没有直接钉住 Qwen2.5 正向 buffer，后者仍是回归缺口。^[PR #5073] - ### EXEC-2a — loader 的 dtype 与 config 获取必须显式、最小化 - 触发：模型 loader 构造 text encoder、VAE、transformer 或只读取 checkpoint config。 @@ -203,29 +157,6 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 记录峰值显存和 dtype。Krea 2 的具体约束见 [Krea 2 规则](../../models/krea2/rules.md)。 ^[PR #4730] -### EXEC-1e — upstream registry 重名时 Omni override 与 plain-vLLM forward 必须同时成立 - -- 触发：上游开始注册 Omni 同名 architecture，或 model runner/forward/capture/dummy-run 接口变化。 -- 强制：全局 registry 无条件重注册 Omni owner；非 staged 模式为 Qwen Omni 选择 thinker，并返回 - stock runner 可消费的 bare tensor；只有 staged talker consumer 存在时才 capture hidden layers。 - runner override 必须接受上游新增 kwargs；text/MoE dummy input 可按 upstream 要求 randomize，但 - code2wav 等结构化 codec id 只能接受参数而不得随机化。worker profiling 同步保存上游新增结果字段。 -- 禁止：因 upstream 已有同名 arch 就跳过 Omni 注册；plain serve 返回 staged tuple；把 vocab-uniform - id 喂给 codec codebook；用 `**kwargs` 隐藏未审查的签名漂移。 -- 验收：registry collision 解析到 Omni class；Qwen2.5/Qwen3 plain thinker 与 staged capture 分别断言 - output shape，PP intermediate 原样透传；dummy-run 覆盖 text randomize 与 generation no-randomize， - 并对 profiling result 做字段 parity。^[PR #5976] - -### EXEC-1f — multimodal wrapper 与 hash algorithm 必须通过 live context 传播 - -- 触发：upstream multimodal input 开始保留原始 bytes、修改 hash 签名或 processor item wrapper。 -- 强制：所有 `get_mm_hashes` / direct `MultiModalHasher.hash_kwargs` 调用传当前 multimodal config 的 - algorithm；processor 通过公开 `get(index)` 解包 `MediaWithBytes`，再把 frames/metadata 交给 HF。 -- 禁止：直接遍历 `.data` 绕过 unwrap；让 stage/replica UUID 前缀替代 content hash algorithm；只修 - 一个模型而不 census 自定义 processor 与 direct hasher caller。 -- 验收：不同 algorithm 进入 hash consumer；wrapped/unwrapped video 都生成同一 HF input，metadata - 保留；stage+replica scope 只包裹 base UUID 且用户 UUID 优先。^[PR #5976] - ### EXEC-2b — fused shard 必须按 source 完整性与布局数值闭环 > H3 text encoder 当前有 fused QKV 与 gate/up owner；Qwen3 code predictor 仍使用分离 @@ -249,32 +180,6 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 encoder 还须对任何未加载 plain retained parameter 在启动时硬失败；unknown checkpoint key 可继续告警，因为它不会留下 model parameter 未初始化。^[PR #4958] ^[PR #5777] ^[PR #5824] -### EXEC-1g — request-end payload 延迟 D2H 必须先取得 device snapshot - -- 触发：模型设置 `omni_payload_at_request_end`，或修改 full-payload accumulation、CUDA graph - 和 D2H 策略。 -- 强制：只在显式 opt-in、无 prefix cache 且下游消费完整 payload 时延迟 D2H；每步 - clone device tensor，相容 list 可 pack 后以 views 复原，结束才跨设备，中间 output 为 `None`。 -- 禁止：累计 graph/input-buffer alias，或扩散到普通逐步 payload。 -- 验收：覆盖 source 被下一 step 覆写、ragged list shape/value/alias、普通路径与 - finish/abort 清理。^[PR #5957] - -### EXEC-1h — pooling stage 的输入输出必须走显式跨 stage bridge - -- 触发：新增非生成 pooling stage，或在 batch pipeline 中跨 stage 传递音频、文本、multimodal payload 与解码结果。 -- 强制：producer 只在上游请求完成后按原 request index 构造输入，分词一次并随音频携带 `aligner_words`、language 和真实 duration；pooling runner 识别 `is_pooling_model`，绕过仅适用于 AR hidden state 的 Omni prefix-cache/mm 提取，decoder 输出固定的 int32 `[n_words, 2]`；最终 consumer 使用同一份 words、duration 和原 batch 映射。 -- 禁止：把 pooling 请求当作 `generate` 或隐式广播单请求 metadata；按 downstream position 压缩原 batch index；重新分词后与 interval 做未验证的 positional zip；把空结果、解码失败或缺失 word metadata 伪装成 `[0, 0]` 的有效对齐；假设真实 `OmniRequestOutput` 暴露测试 fake 才有的 `additional_information`。 -- 验收：用真实 stage wrapper、`PoolingOutput` 和 `OmniRequestOutput` 做 mixed-batch 测试，分别断言音频完成门槛、metadata/shape、duration clamp、原 request 标签和 mismatch→无时间戳；同时验证普通生成 stage 的输入输出不受影响。 -^[PR #4795] - -### EXEC-1i — 有状态跨 stage AR bridge 必须按 request 累积并完整消费 payload - -- 触发：多 stage AR 模型在 `preprocess` 中按 request 保存 KV、音频帧或 TTS 状态，并通过 connector 传递累积 timeline/code payload。 -- 强制：以 request id 隔离 session；无法证明 batch-safe 时显式限制 `max_num_seqs=1` 并在 request 完成/abort 时清理；下游每次采用累积 payload，单次 wake drain 所有尚未消费的位置，prompt-region 状态也必须先推进；code2wav 只接受真实 `[frames, 31]` codec codes。 -- 禁止：按一个 wake 只执行一个新位置；把延迟或合并的 chunk 当作只含一个 token；缺失 connector payload 时把 placeholder `input_ids` 当 codec codes 解码；用静默截断或 clamp 掩盖错误 shape/range。 -- 验收：覆盖 delayed/coalesced chunks、首个 PAD prompt chunk、zero-progress wake、最终 marker、缺失 payload、flat/二维 code shape、越界 code 与 session cleanup；断言 talker 与 code2wav 的真实输出字段和音频类型。 -^[PR #5842] - ### EXEC-6a — shared image example 先建 canonical envelope，model-extra 只做特化变换 - 触发：修改 shared T2I/I2V example、`model_extras` prompt registry 或模型 prompt builder。 diff --git a/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md b/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md index 46cfd8a78..7228585fb 100644 --- a/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md +++ b/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md @@ -1,7 +1,7 @@ --- title: "IndexTTS 2.5 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models] sources: ["PR #5957", vllm_omni/deploy/indextts2_5.yaml, vllm_omni/model_executor/models/indextts2/, vllm_omni/model_executor/stage_input_processors/indextts2.py, vllm_omni/entrypoints/openai/tts_adapters/indextts2.py, tests/model_executor/models/indextts2/, tests/entrypoints/openai_api/test_tts_adapter.py, tests/entrypoints/openai_api/test_serving_speech.py] @@ -40,7 +40,7 @@ confidence: high uploaded voice 都覆盖。 request-end device buffer 的共享生命周期只由 -[EXEC-1g](../../components/model-executor/rules.md#exec-1g-request-end-payload-延迟-d2h-必须先取得-device-snapshot) +[EXEC-1g](../../components/model-executor/rules-bridge-batch.md#exec-1g-request-end-payload-延迟-d2h-必须先取得-device-snapshot) 定义，本 owner 只声明 IndexTTS 2.5 是显式 request-end consumer。 ## ITTS25-1c — 性能与质量证据必须保留可复核边界 diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index cf1ba77a9..74d71078f 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -1,7 +1,7 @@ --- title: "Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS）" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-04 type: index tags: [vllm-omni, models, qwen-omni] sources: ["PR #5073", "PR #5671", "PR #5976", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, docs/design/qwen3_omni_tts_performance_optimization.md] @@ -27,7 +27,7 @@ sources: ["PR #5073", "PR #5671", "PR #5976", vllm_omni/model_executor/models/qw - vLLM 0.27 后 upstream 与 Omni registry 有同名 architecture：Omni 必须覆盖全局 entry；plain `vllm serve` 缺 `model_stage` 时 Qwen2.5/Qwen3 默认 thinker，Qwen3 thinker 在非 staged 模式返回 bare tensor，只有 staged talker consumer 才请求 captured layers；共享验收见 - [EXEC-1e](../../components/model-executor/rules.md#exec-1e-upstream-registry-重名时-omni-override-与-plain-vllm-forward-必须同时成立)。^[PR #5976] + [EXEC-1e](../../components/model-executor/rules-bridge-batch.md#exec-1e-upstream-registry-重名时-omni-override-与-plain-vllm-forward-必须同时成立)。^[PR #5976] - 官方历史文档：`docs/design/module/archive/async_omni_architecture.md`（以 Qwen3-Omni 为 worked example 的分层运行时快照，非 active spec）、 `docs/design/qwen3_omni_tts_performance_optimization.md`（性能优化实录） diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md index d19f46b2f..7c3cf492a 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md @@ -1,7 +1,7 @@ --- title: "Qwen-Omni 家族拓扑与性能结论" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-04 type: architecture tags: [vllm-omni, models, qwen-omni] sources: ["PR #5073", "PR #5671", docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/config/model.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/registry.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/worker/gpu_model_runner.py] @@ -38,7 +38,7 @@ sources: ["PR #5073", "PR #5671", docs/design/qwen3_omni_tts_performance_optimiz checkpoint metadata 与排除规则，不能泛化成所有 Qwen2.5 stage 或任意 FP4 checkpoint。 - Qwen2.5 Talker 的外部 3584 width 在 forward 内投影到内部 896；shared runner 的 buffer 必须按 `get_inputs_embeds_size()` 分配。Qwen3 在 preprocessing 已投影，未 override 时仍回退 internal - hidden size。共享门禁见 [EXEC-1d](../../components/model-executor/rules.md#exec-1d-cross-stage-embedding-buffer-必须按-ingress-width-分配)。 + hidden size。共享门禁见 [EXEC-1d](../../components/model-executor/rules-bridge-batch.md#exec-1d-cross-stage-embedding-buffer-必须按-ingress-width-分配)。 - 最终 head 的外部验证只覆盖单张 RTX PRO 6000 Blackwell、TP1、特定 vLLM/PyTorch/ Transformers/FlashInfer 栈：full three-stage server 启动、仅 Thinker 选 NVFP4 kernel，以及 deterministic text smoke。accuracy/perf 也是同一硬件上的 text-output workload，且相对 BF16 From f2019d6bea3f5116213c61b7a84a8acaad3bfb84 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 02:43:03 +0800 Subject: [PATCH 124/465] knowledge(vllm-omni): sync 208c6776 (PR #5068) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 208c6776a89f9dfa1598457196d71538bb9b24ac Upstream-PR: #5068 Knowledge-Delta: 新增 EXEC-8a，记录 GLM-TTS eager block-causal Euler 循环中不变 DiT 条件与 CFG batch 必须按请求预计算并校验其长度、设备和 dtype。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5b04cc124..bda764bb6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 38a2500bdf887cd0d23c1c1616c59faec6ffd317 + previous_audited_sha: fcf1085395940df1f9486e2f1b6f11581d8002c9 audited_ref: main - audited_sha: fcf1085395940df1f9486e2f1b6f11581d8002c9 + audited_sha: 208c6776a89f9dfa1598457196d71538bb9b24ac # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b89a973c9..e6963d661 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -fcf1085395940df1f9486e2f1b6f11581d8002c9`. The canonical machine baseline +208c6776a89f9dfa1598457196d71538bb9b24ac`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5c5c07da9..ce21a31ab 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fcf10853` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 208c6776` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 87f489a9b..2f10399ab 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fcf10853` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 208c6776` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2158e1a70..01ffcb1fd 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fcf10853` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 208c6776` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5efb4fdce..ed8b1ab73 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fcf10853` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 208c6776` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index b54f67ef4..beae93dfb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068"] --- # Model Executor 规则 @@ -199,3 +199,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：`OmniRequestOutput` 必须继承 vLLM `RequestOutput` 并将继承字段作为 dataclass 字段直接声明；stage 原始输出必须通过 `from_stage_output()` 显式复制 `prompt`、`prompt_token_ids`、`outputs`、`finished` 等生成内容，源对象为 `OmniRequestOutput` 时再复制 diffusion 内容；消费者直接读取扁平字段，列表归一化只在真实列表边界完成。 - 禁止：重新引入 `request_output` 嵌套字段、动态 pass-through property、修改 dataclass 生成的 `__init__` 或递归 `unwrap` 兼容层；不得在 serving、example 或工具代码中恢复 `output.request_output` 访问。 - 验收：用真实 `RequestOutput` 和 `OmniRequestOutput` 分别验证 factory 复制的 prompt、token、outputs、finished 及 images/trajectory 内容；覆盖文本、音频、图像和 pipeline stage 的直接字段消费，并确认未完成 stage 的 `finished` 不被包装过程改写。^[PR #5146] + +### EXEC-8a — DiT Euler 采样循环必须只计算一次不变条件 + +- 触发：修改 GLM-TTS flow-matching DiT 的 Euler 采样循环、文本 embedding、RoPE、block-causal mask、CFG batch，或新增预计算输入参数。 +- 强制：在每次 `_do_sample` 调用中只计算一次不随 timestep 或采样状态变化的 text embedding、RoPE、attention mask 和 CFG 双 batch 张量，并传入 DiT；同时保持 `seq_len`、设备、dtype 与 `forward()` 的合同一致，缺省参数仍保留原有调用行为。 +- 禁止：在每个 Euler step 重复构造上述固定输入；让预计算 embedding 或 RoPE 在长度不匹配时静默继续；改变原有 mask device、模型 dtype 或 block-causal 语义；把 eager streaming 路径的优化结论外推为 CUDA-graph 路径或其他模型的通用结论。 +- 验收：覆盖 CFG/non-CFG 和 block-causal streaming 路径，验证预计算输入的长度、设备、dtype 与采样序列匹配，并以数值/字节一致性、输出 shape 及 profiling 证明循环内不再重复执行固定计算；确认非 streaming CUDA-graph 和未提供预计算参数的旧调用仍正常。 ^[PR #5068] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d1fea376d..d8ebcfa1a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fcf10853` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 208c6776` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 785270b53..c289fc464 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fcf10853` 验证存在 +- 源码校验：以上路径均已在 `main @ 208c6776` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1c2b65e63..016d71375 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fcf10853`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 208c6776`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `38a2500b` 本轮新增 #5146 的统一输出对象合同：`OmniRequestOutput` 归入 model-executor，扁平对象的 msgpack 传输与旧嵌套线格式兼容归入 distributed。不要据此推断模型专有 modality 字段已经完成重构，也不要把示例、测试迁移或单次 round-trip 证据解释为所有模型和硬件的端到端 parity。 +本次相对 `fcf10853` 本轮将 GLM-TTS 流式/分块因果路径中 Euler 循环的固定条件 hoist 合同收录到 model-executor 规则，新增 EXEC-8a 并补充 PR #5068 来源。该规则不表示所有模型或所有采样路径都应采用相同优化，也不保证不同硬件和并发度下都有相同收益。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 38a2500b \ - --to fcf10853 \ + --from fcf10853 \ + --to 208c6776 \ --repo \ --mode report-only ``` From 778868a5ab06bb6cc06e497c30f8b1acbf3c7338 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 02:44:14 +0800 Subject: [PATCH 125/465] knowledge(vllm-omni): sync e6773601 (PR #6107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: e677360168d7d4f512bcebd50fbb1cbfc422d133 Upstream-PR: #6107 Knowledge-Delta: 新增 Cosmos3 默认 guardrails 的依赖声明与缺包 fail-closed 验收规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/cosmos3/rules.md | 11 +++++++++-- 10 files changed, 22 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bda764bb6..bbfb6189f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fcf1085395940df1f9486e2f1b6f11581d8002c9 + previous_audited_sha: 208c6776a89f9dfa1598457196d71538bb9b24ac audited_ref: main - audited_sha: 208c6776a89f9dfa1598457196d71538bb9b24ac + audited_sha: e677360168d7d4f512bcebd50fbb1cbfc422d133 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e6963d661..324fd4621 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -208c6776a89f9dfa1598457196d71538bb9b24ac`. The canonical machine baseline +e677360168d7d4f512bcebd50fbb1cbfc422d133`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ce21a31ab..82b5161b2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 208c6776` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e6773601` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2f10399ab..169d70307 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 208c6776` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e6773601` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 01ffcb1fd..d7fb7c4de 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 208c6776` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e6773601` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ed8b1ab73..4f79c5c33 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 208c6776` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e6773601` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d8ebcfa1a..7e8372e7f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 208c6776` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e6773601` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c289fc464..b6f2ff0b5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 208c6776` 验证存在 +- 源码校验：以上路径均已在 `main @ e6773601` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 016d71375..a78cae7dd 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 208c6776`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e6773601`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `fcf10853` 本轮将 GLM-TTS 流式/分块因果路径中 Euler 循环的固定条件 hoist 合同收录到 model-executor 规则，新增 EXEC-8a 并补充 PR #5068 来源。该规则不表示所有模型或所有采样路径都应采用相同优化，也不保证不同硬件和并发度下都有相同收益。 +本次相对 `208c6776` 本轮将 Cosmos3 默认启用 guardrails 所需的 `cosmos-guardrail` 依赖与缺包时的 fail-closed 错误合同沉淀到 Cosmos3 规则页。该规则不表示 HF authentication、模型质量、性能或其他 Cosmos3 变体能力已得到验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from fcf10853 \ - --to 208c6776 \ + --from 208c6776 \ + --to e6773601 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index e9c2b32b9..448063701 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -1,10 +1,10 @@ --- title: "Cosmos3 规则" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107"] confidence: high --- @@ -103,3 +103,10 @@ confidence: high 共享 RNG/graph 规则见 [Diffusion rules](../../components/diffusion/rules.md)；公开证据分层见 [model adaptation guardrails](../../review/guides/model-adaptation-guardrails.md)。 + +## COSMOS-4a — 默认 guardrails 必须具备依赖并在缺包时 fail closed + +- 触发：Cosmos3 以默认开启的 guardrails 加载或运行，或修改 Cosmos3 guardrail 依赖与缺包处理。 +- 强制：在公共依赖中声明 `cosmos-guardrail>=0.3.1`，使标准 vLLM-Omni 镜像具备默认 guardrails 所需包；guardrail import 失败时必须立即抛出明确的 `ValueError`，说明 guardrails 已启用但不可用、缺少 `cosmos-guardrail`，并保留 NVIDIA Open Model License Agreement 合规提示。 +- 禁止：依赖用户手动安装额外包才能获得默认行为；缺少 guardrail 包时静默继续、把状态描述为用户主动禁用了 safety checker，或把安装成功外推为 HF authentication、模型质量和其他变体能力已验证。 +- 验收：干净环境安装公共 requirements 后 `cosmos-guardrail` 可导入，Cosmos3 默认 guardrails 路径可进入模型运行；模拟缺少该包时启动/构造 fail fast，错误文本同时包含包名和许可证链接。^[PR #6107] From 8d165462da24fc0804e5b077d0bfb885fcf286ef Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 02:45:40 +0800 Subject: [PATCH 126/465] knowledge(vllm-omni): sync 5358f5db (PR #5174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 5358f5db01e512dc20fcc29a1d122e82c8546780 Upstream-PR: #5174 Knowledge-Delta: 新增 OmniVoice 热路径审查组 EXEC-9a，固化固定输入缓存、批量 D2H、attention mask 语义与 TF32 opt-in 验收边界。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bbfb6189f..3952b8972 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 208c6776a89f9dfa1598457196d71538bb9b24ac + previous_audited_sha: e677360168d7d4f512bcebd50fbb1cbfc422d133 audited_ref: main - audited_sha: e677360168d7d4f512bcebd50fbb1cbfc422d133 + audited_sha: 5358f5db01e512dc20fcc29a1d122e82c8546780 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 324fd4621..909787c74 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e677360168d7d4f512bcebd50fbb1cbfc422d133`. The canonical machine baseline +5358f5db01e512dc20fcc29a1d122e82c8546780`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 82b5161b2..0ff2d63e1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e6773601` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5358f5db` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 169d70307..aa45398de 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e6773601` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5358f5db` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d7fb7c4de..41b498514 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e6773601` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5358f5db` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4f79c5c33..55aa54c12 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e6773601` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5358f5db` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index beae93dfb..06ad2d9f9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174"] --- # Model Executor 规则 @@ -206,3 +206,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：在每次 `_do_sample` 调用中只计算一次不随 timestep 或采样状态变化的 text embedding、RoPE、attention mask 和 CFG 双 batch 张量，并传入 DiT；同时保持 `seq_len`、设备、dtype 与 `forward()` 的合同一致，缺省参数仍保留原有调用行为。 - 禁止：在每个 Euler step 重复构造上述固定输入；让预计算 embedding 或 RoPE 在长度不匹配时静默继续；改变原有 mask device、模型 dtype 或 block-causal 语义；把 eager streaming 路径的优化结论外推为 CUDA-graph 路径或其他模型的通用结论。 - 验收：覆盖 CFG/non-CFG 和 block-causal streaming 路径，验证预计算输入的长度、设备、dtype 与采样序列匹配，并以数值/字节一致性、输出 shape 及 profiling 证明循环内不再重复执行固定计算；确认非 streaming CUDA-graph 和未提供预计算参数的旧调用仍正常。 ^[PR #5068] + +### EXEC-9a — OmniVoice 热路径的固定输入缓存与掩码/精度边界 + +- 触发：修改 OmniVoice 离散扩散生成循环中的 D2H 同步、attention mask、RoPE、文本/音频 embedding、CFG、CUDA graph/eager 路径或 TF32 配置。 +- 强制：将循环不变的文本 embedding、`audio_mask_3d`、RoPE 和 additive attention mask 在循环外缓存；批量取得 `c_lens` 并只在实际消费的 logits slice 上转为 fp32；CUDA graph 与 eager 使用相同的 float mask 语义，bool mask 必须将 True 映射为 `0.0`、False 映射为 `-inf`，已有 float mask 必须原样保留；TF32 只能通过显式配置 opt-in，并在 graph capture 前启用。 +- 禁止：每步重复计算固定输入或执行逐请求 D2H `.item()`；对 float mask 使用布尔反转或把 bool 直接隐式转换进 float buffer；让 graph capture 与 replay 使用不同 mask dtype；默认开启或宣称 TF32 路径 bit-identical；只更新 conditional 或 unconditional 一侧的迭代 token。 +- 验收：覆盖 eager 与 CUDA graph、bool 与 additive float mask、CFG/non-CFG 及不同 batch/concurrency，断言 masked 位置仍为 `-inf`、float mask 数值保留、graph/eager 输出满足既定精度合同、每步无固定输入重算和逐请求同步；单独验证 `enable_tf32` 默认关闭且显式开启前后性能/非 bit-identical 语义有记录。 ^[PR #5174] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7e8372e7f..cefaa0a79 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e6773601` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5358f5db` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b6f2ff0b5..6116823fe 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e6773601` 验证存在 +- 源码校验：以上路径均已在 `main @ 5358f5db` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a78cae7dd..1f1467fd7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e6773601`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5358f5db`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `208c6776` 本轮将 Cosmos3 默认启用 guardrails 所需的 `cosmos-guardrail` 依赖与缺包时的 fail-closed 错误合同沉淀到 Cosmos3 规则页。该规则不表示 HF authentication、模型质量、性能或其他 Cosmos3 变体能力已得到验证。 +本次相对 `e6773601` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 208c6776 \ - --to e6773601 \ + --from e6773601 \ + --to 5358f5db \ --repo \ --mode report-only ``` From 8e3d69a4668a12e39fb7154034673d6442ed15da Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 02:47:01 +0800 Subject: [PATCH 127/465] knowledge(vllm-omni): sync 72a02b49 (PR #6121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 72a02b492a6957acc4659fde09be85c00ad0f90b Upstream-PR: #6121 Knowledge-Delta: 新增 SERV-5g：pure-diffusion speech factory 绕过 __init__ 时，必须在 AR TTS adapter 解析和 _adapter 访问前按 _diffusion_mode 短路，并分别验证 diffusion 与普通 AR 路径。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 11 +++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 22 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3952b8972..7d295ba60 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e677360168d7d4f512bcebd50fbb1cbfc422d133 + previous_audited_sha: 5358f5db01e512dc20fcc29a1d122e82c8546780 audited_ref: main - audited_sha: 5358f5db01e512dc20fcc29a1d122e82c8546780 + audited_sha: 72a02b492a6957acc4659fde09be85c00ad0f90b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 909787c74..2c7571975 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5358f5db01e512dc20fcc29a1d122e82c8546780`. The canonical machine baseline +72a02b492a6957acc4659fde09be85c00ad0f90b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0ff2d63e1..a10d3fc87 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5358f5db` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 72a02b49` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index aa45398de..8bec873b2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5358f5db` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 72a02b49` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 41b498514..d63fa5a06 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5358f5db` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 72a02b49` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 55aa54c12..6a89c883b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5358f5db` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 72a02b49` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cefaa0a79..875ae72fe 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5358f5db` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 72a02b49` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6116823fe..aae75ecd2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5358f5db` 验证存在 +- 源码校验：以上路径均已在 `main @ 72a02b49` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 5e13c3b02..1024f096d 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -1,10 +1,10 @@ --- title: "Serving engine 生命周期规则" created: 2026-09-03 -updated: 2026-09-03 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121"] confidence: high --- @@ -93,6 +93,13 @@ confidence: high ## Full-duplex 与 CFG companion 生命周期 +### SERV-5g — pure-diffusion TTS 必须在 AR adapter 解析前短路 + +- 触发：speech serving 同时支持普通 AR TTS 与 pure-diffusion TTS，且 diffusion factory 通过 `for_diffusion()`/`__new__` 绕过常规 `__init__`，发生 adapter 解析或 native speed handling 变更。 +- 强制：在读取 adapter 或 `_adapter` 状态前先检查在两种构造路径上都可用的 `_diffusion_mode`；pure-diffusion 路径必须直接返回 `None` 并保持专用 speech 编码路径，普通 AR 实例继续通过 `resolve_adapter()` 使用既有 adapter-native speed 行为。 +- 禁止：在 diffusion 实例上无条件解析 AR-stage TTS adapter、读取仅由 `__init__` 设置的 `_adapter`，或把 pure-diffusion 请求重新路由到 AR adapter 路径；只覆盖普通构造器而不覆盖 bypass-`__init__` factory。 +- 验收：用 `OmniOpenAIServingSpeech.for_diffusion()` 构造实例，提交公开 `create_speech()` 请求并断言成功的 `200`/`audio/wav` 响应；以 mock 断言 diffusion speed 请求返回原值且 `resolve_adapter()` 未被调用，同时验证普通 AR 实例仍解析并使用其 adapter，所有路径不得出现 `AttributeError`。^[PR #6121] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1f1467fd7..8497bb0e1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5358f5db`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 72a02b49`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `e6773601` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `5358f5db` 本轮为 serving 组件补充纯 diffusion TTS 的 adapter 边界规则，并把 PR #6121 记入该规则页来源。测试文件的归属是 testing/ci，超出本轮允许写入范围；本条规则不能推断 OmniVoice GPU E2E 已通过，或证明所有 bypass-`__init__` 属性都已完整初始化。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from e6773601 \ - --to 5358f5db \ + --from 5358f5db \ + --to 72a02b49 \ --repo \ --mode report-only ``` From db43bbc3727960f05eaf38c9d672d8cb347497ef Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 02:47:32 +0800 Subject: [PATCH 128/465] knowledge(vllm-omni): sync 12f81681 (PR #5656) Upstream-Commit: 12f81681793441632ffb7d68b200b9d61821e1f8 Upstream-PR: #5656 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7d295ba60..d32a776c6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5358f5db01e512dc20fcc29a1d122e82c8546780 + previous_audited_sha: 72a02b492a6957acc4659fde09be85c00ad0f90b audited_ref: main - audited_sha: 72a02b492a6957acc4659fde09be85c00ad0f90b + audited_sha: 12f81681793441632ffb7d68b200b9d61821e1f8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2c7571975..c63ea1a41 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -72a02b492a6957acc4659fde09be85c00ad0f90b`. The canonical machine baseline +12f81681793441632ffb7d68b200b9d61821e1f8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a10d3fc87..b3a2fa14d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 72a02b49` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 12f81681` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8bec873b2..12a0db955 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 72a02b49` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 12f81681` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d63fa5a06..c786dd0cd 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 72a02b49` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 12f81681` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6a89c883b..6284292a6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 72a02b49` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 12f81681` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 875ae72fe..c572453a6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 72a02b49` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 12f81681` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index aae75ecd2..f307bc64a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 72a02b49` 验证存在 +- 源码校验：以上路径均已在 `main @ 12f81681` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8497bb0e1..c0aa6ecc1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 72a02b49`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 12f81681`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `5358f5db` 本轮为 serving 组件补充纯 diffusion TTS 的 adapter 边界规则，并把 PR #6121 记入该规则页来源。测试文件的归属是 testing/ci，超出本轮允许写入范围；本条规则不能推断 OmniVoice GPU E2E 已通过，或证明所有 bypass-`__init__` 属性都已完整初始化。 +本次相对 `72a02b49` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 5358f5db \ - --to 72a02b49 \ + --from 72a02b49 \ + --to 12f81681 \ --repo \ --mode report-only ``` From 094123e9c4cf7dbfb54a1e393672ab10823e4ea1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 03:42:58 +0800 Subject: [PATCH 129/465] adapters(vllm-omni): ignore the renamed .readthedocs.yml Unit 125 halted on a scheduled decision: PR #6129 renames .readthedocs.yaml to .readthedocs.yml, and only the old spelling was in ignored_paths, so the new one came through as an unmatched_path with no owner. This is a policy-map change, which the campaign deliberately never makes on its own -- the driver halts and a human decides -- so it lands here as a non-unit commit. Documentation tooling is ignored by the same reasoning that already ignores the .yaml spelling. preflight.py regenerated: scheduled decisions are now 220, 232, 259 and 294. No unit is consumed and audited_sha is untouched. Release audit CLEAN. Co-Authored-By: Claude Opus 5 (1M context) --- adapters/vllm_omni/release_baseline.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d32a776c6..ecd8db870 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -118,6 +118,8 @@ ignored_paths: reason: repository tooling - pattern: .readthedocs.yaml reason: documentation tooling +- pattern: .readthedocs.yml + reason: documentation tooling（上游在 PR #6129 改了扩展名） - pattern: CODE_OF_CONDUCT.md reason: community metadata - pattern: CONTRIBUTING.md From 53e3a604efd8a98fe593a3b168661af889556e3c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 03:55:13 +0800 Subject: [PATCH 130/465] knowledge(vllm-omni): split the cache and task group out of minimax-h3 rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit split_risk.py named this page as the next to hit the 32768-byte gate: 1991 bytes of headroom with 29 remaining units routing to minimax-h3, roughly a quarter of the run. Splitting it now is the difference between scheduled work and another halt mid-run, which is what unit 120 cost. The page already delegates by topic -- MMH3-1a/1f to loading, 2a/2b to media, 3x to deployment -- so the split follows that shape rather than inventing one: MMH3-2c-2h -> rules-cache-task.md conditioned VAE determinism, modular task selection, and request-scoped Cache-DiT, TeaCache and distilled sigma schedule Rule IDs unchanged; the Direct 快速入口 rows now link the extracted page. Headroom after: rules.md 14127 B, the new page 19443 B. One inbound anchor pointed at the moved MMH3-2g, from components/diffusion/cache-acceleration.md, and was retargeted. This is the third split in a row to break an anchor that check_knowledge_tree cannot see -- it validates file existence, not fragments -- so the grep is now part of the split procedure rather than something to rediscover. minimax-h3/_index.md registers by prose rather than a table, so the entry goes in the 审查入口 paragraph alongside the other three siblings. No unit is consumed and audited_sha is untouched. Gates: tree 0 errors / 12 pre-existing reminders, wiki lint 0 errors across 327 pages, release audit CLEAN, test_knowledge_source 12 passed. The six quick-entry rows carry the moved rule fragment, not just the page. Linking the page root would land a Direct reviewer at the top of a 160-line file instead of the rule they selected -- the same anchor repair the split owes its inbound links, applied to its own outbound ones. The generated MMH3-2g anchor matches the one already used from cache-acceleration.md, which confirms the slug convention. Co-Authored-By: Claude Opus 5 (1M context) --- .../diffusion/cache-acceleration.md | 4 +- .../vllm-omni/models/minimax-h3/_index.md | 4 +- .../models/minimax-h3/rules-cache-task.md | 160 ++++++++++++++++++ .../vllm-omni/models/minimax-h3/rules.md | 154 +---------------- 4 files changed, 170 insertions(+), 152 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md diff --git a/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md b/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md index ce4042a33..2611dc14e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md +++ b/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md @@ -1,7 +1,7 @@ --- title: "缓存加速（cache_dit / TeaCache / stepcache / magcache）" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-04 type: guide tags: [vllm-omni, components, diffusion] sources: ["PR #5840", docs/design/feature/cache_dit.md, docs/design/feature/teacache.md, recipes/MiniMaxAI/MiniMax-H3.md, vllm_omni/diffusion/cache/, vllm_omni/diffusion/data.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, tests/diffusion/cache/test_teacache_extractors.py] @@ -32,7 +32,7 @@ sources: ["PR #5840", docs/design/feature/cache_dit.md, docs/design/feature/teac "AudioXPipeline"}`——这两条 pipeline 不支持 cache_dit/tea_cache。 - MiniMax-H3 只校准 FL2VA：Ref2VA-only fail fast，combined 只 hook `transformer` 并告警 `transformers_ref` 不缓存；具体阈值与证据见 - [MMH3-2g](../../models/minimax-h3/rules.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state)。 + [MMH3-2g](../../models/minimax-h3/rules-cache-task.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state)。 ## 相关 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 757dbf01b..d4cf6245a 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -1,7 +1,7 @@ --- title: "MiniMax H3" created: 2026-08-05 -updated: 2026-09-02 +updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] @@ -111,4 +111,4 @@ T2VA 因当前严格 gate 未达标被移出；这表示“无 retained pixel-le H3 input matrix/media ingress，以及 text-encoder completeness、online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 [media rules](rules-media.md) 与 [MiniMax H3 rules](rules.md#direct-代码快速入口)；checkpoint transform、quantized loader 与 text-encoder fused-source 完整性正文见 [loading rules](rules-loading.md)；DLO、consumer/H100/ROCm -部署和硬件证据正文见 [deployment rules](rules-deployment.md)。 +部署和硬件证据正文见 [deployment rules](rules-deployment.md)；conditioned VAE 确定性、modular task 选择与 request 级 Cache-DiT/TeaCache/sigma schedule 生命周期见 [缓存与任务生命周期规则](rules-cache-task.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md new file mode 100644 index 000000000..af648c5f6 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md @@ -0,0 +1,160 @@ +--- +title: "MiniMax H3 缓存与任务生命周期规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5703", "PR #5720", "PR #5837", "PR #5840", "PR #5853", "PR #5991", vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py] +confidence: high +--- + +# MiniMax H3 缓存与任务生命周期规则 + +`MMH3-2c`–`MMH3-2h`：conditioned VAE 的确定性、modular task 选择，以及 request 级 +Cache-DiT、TeaCache 与 distilled sigma schedule 的生命周期。触发信号见 +[MiniMax H3 规则](rules.md) 的 Direct 代码快速入口；加载合同见 +[加载规则](rules-loading.md)，媒体输入见 [媒体规则](rules-media.md)。 + +## MMH3-2c — conditioned VAE 的固定种子必须按实际设备隔离并恢复 + +- 触发：修改 H3 image/video reference 的 VAE encode、固定 keyframe seed、`fork_rng`、设备 + generator 或 accelerator backend 支持。 +- 强制：`encode_image()`/`encode_video()` 在采样前暂时把 VAE 转成 FP32，在 + `torch.random.fork_rng` 同时传 `devices=...` 与 `device_type=parameter.device.type`，并在 + context 内播种 CPU default generator 与 active-device generator；退出时恢复 RNG state， + 在 `finally` 恢复原 dtype，image path 还必须恢复 `parallel_tiling`。这个 seed 是 + conditioned VAE 的固定内部 seed，不能描述成 request seed。 +- 强制：`devices` 只决定保存/恢复哪些 device index，不能代替 `device_type` 选择 RNG module； + 目标 torch 版本省略后者会默认进入 CUDA。NPU 依赖 `torch_npu` 先注册 `torch.npu`，encode + 内的 `self.device_module` 则在同一 active device 上执行 `device(...)` 与 `manual_seed(...)`。 +- 禁止：把 PR 文本中的 CUDA+MUSA allowlist 当成目标实现。目标代码实际以 + `parameter.device.type != "cpu"` 接纳已注册的 accelerator device module，并把真实 type + 交给 `fork_rng`；CPU 的 + `devices=[]` 仍保存/恢复 CPU RNG。实机证据覆盖 CUDA/MUSA，以及 PR #5837 报告的 Ascend + NPU direct fork smoke 与一次双参考图 FL2VA serving 成功，不能外推 XPU 或 ROCm。也不能因 + `fork_rng` 最终恢复 state 就宣称并发安全：context 内仍会暂时改写 process-global CPU/device + generator，重叠调用需要序列化或独立并发证明。 +- 验收：同 seed 的 image/video condition latent 可重复，正常和 encode 异常后 CPU、目标设备 + RNG state、dtype 及 image tiling state 都恢复；CPU 与每个声称支持的 accelerator 分支分别 + 覆盖，并加入重叠调用 fence。PR 中拟议的专用 VAE 单测按 review 被删除，目标 commit 没有 + 新增测试；PR #5837 也只改两处参数，未提交 CPU/NPU 回归测试，其 NPU smoke、serving 结果 + 与 CUDA 不变性说明只能作为外部证据，不能冒充持续回归覆盖。^[PR #5703] ^[PR #5837] + +## MMH3-2d — modular H3 的 task selector 必须同步权重、能力与所有 DiT lifecycle + +- 触发：修改 `modular_model_index.json`、`MiniMaxH3ModularPipeline`、`--task-type`、combined + FL2VA/Ref2VA 服务、`_dit_modules` 或 shared text/VAE components。 +- 强制：模型发现同时识别 `model_index.json` 与 `modular_model_index.json`，registry alias 与 + postprocess 都映射到同一 H3 pipeline。`auto/combined` 从 root 加载 FL2VA `transformer` 与 + Ref2VA `transformers_ref`，但共享 text encoder、video/audio VAE；单 task 只加载所需 DiT。 + request task 必须属于启动时 `supported_tasks`；combined 省略 task 时按无媒体→t2va、image-only + →fl2va、video/audio→ref2va 推断，因此 image-only Ref2VA 必须显式给 task；Ref2VA-only + 省略 task 时保留 implicit ref2va。 +- 强制：modular alias 复制 9-image/mixed-reference admission capability；所有基于 + `_dit_modules` 的 consumer（loader strictness、Cache-DiT enable/refresh/summary、LoRA/offload） + 遍历实际 DiT，不硬编码 `transformer`/`transformer_2`。共享 `--task-type` 放宽后，非 H3 + owner 仍 model-aware 校验；Qwen3-TTS 只接受既有三值。 +- 缺口：目标 pin 的 modular metadata 虽修复 admission 字段，仍遗漏 canonical H3 的 + `attention_mask_free=True`。HF root 的两个 index 都解析成 modular alias，因此默认 + combined 服务在 Blackwell 不会自动选 TRTLLM，与 recipe 声明冲突；alias 必须校验全部能力 + 字段 parity，不能只复制 review 点名的字段。 +- 禁止：从 combined 注册推断双 DiT 性能已验证；recipe 最终只描述配置，没有 combined + warm latency/throughput/output-parity qualification。也不能用单 DiT Cache-DiT 测试证明 + `transformers_ref` lifecycle。 +- 验收：覆盖 local/Hub index discovery、snapshot allow-pattern、combined/单分区初始化、缺 + partition、task default/membership、modular admission、两 DiT cache lifecycle 与非法 TTS + selector。PR body 的顺序 T2VA→Ref2VA 视频使用 `duration=2.0`，低于目标 pin 已生效的 4 秒 + 下限，因此不能证明 final merge 的 combined 路径；148.27/158.67 GiB combined 与 86.27 GiB + single-load 等数字来自缺硬件/协议/重复次数的 issue comment，不能作为容量保证。^[PR #5720] + +## MMH3-2e — quality 映射只决定 request 的 Cache-DiT 目标 + +- 触发：修改公共 `quality` 值、H3 quality policy、startup cache adoption 或 denoise 前 prepare。 +- 强制：公共层只接受 `None`、`lossless`、`high`，并保留 omitted 与 explicit lossless 的区别。 + 对 request-scoped Cache-DiT，`lossless` 产生空 target；`high` 总是选择 model-owned conservative + profile，即使启动时没有 cache backend；omitted 仅在启动配置为 Cache-DiT 时恢复 server generic + profile，否则不安装 Cache-DiT。 + pipeline 必须在参数/任务/step 解析完成后、真实 denoise 紧邻之前 apply plan。 +- 禁止：沿用 PR 早期描述，把无 startup cache 的 `high` 拒绝；让 unsupported quality 静默落入 + lossless；在 request 未显式提供 quality 时由 sync/streaming serving 覆盖 model default。recipe + 把同一组 quality 数字标为 4×H200，而 PR evidence 把它归于 4×L20X SP4；hardware provenance + 未统一前不得引用该表作为任一硬件的可靠 benchmark，更不能外推通用 speedup/quality。 +- 验收：offline、chat、sync video、streaming video 都覆盖 omitted/lossless/high/非法值及显式字段 + 传播；startup cache on/off × 三种 intent 验证 exact installation key，并断言 prepare 先于 diffuse。 + 当前测试广泛覆盖 validation/routing/policy 和 mock 顺序，但真实 Cache-DiT transition 的并发、取消、 + failure rollback 仍缺证据；独立 reviewer 的单卡 B300 观察也显示 `high` 的质量/延迟与四卡表明显 + 不同，只能支持 topology-dependent 边界。共享状态机与 batch 合同见 + [DIFF-2i–2j](../../components/diffusion/rules-component-lifecycle.md)。^[PR #5853] + +## MMH3-2f — force-refresh hint 属于 active profile identity + +- 触发：H3 request `extra_args` 使用 `force_refresh_step_hint` 或 `force_refresh_step_policy`。 +- 强制：hint 是 1-based positive integer 且不超过 `num_inference_steps`，policy 只接受 once/repeat， + 省略 policy 默认 once；没有 active cache target 时两者都拒绝。用 dataclass replace 生成 request-local + config，不修改 startup generic config；installation key 必须包含 hint+policy，因为 Cache-DiT 的 + incremental refresh 把 `None` 解释为保留旧 hint，变更或移除 hint 必须 reinstall hooks。 +- 禁止：接受 bool 作为整数；只给 policy；跨 request 原地修改共享 config；same-key repeated request + 忘记重置 once hint，导致它只在首个 request 生效。 +- 验收：边界 1/steps、0/越界/bool/非法 policy、无 target、hint add/change/remove、once/repeat 和 + repeated same-key request；断言每次 refresh 重建 hint context、generic config 不变，key 变化发生 + disable→enable。当前覆盖为 mock/config 合同，未证明不同 topology 下 hint 对质量/命中率的效果。 + ^[PR #5853] + +## MMH3-2g — H3 TeaCache 只绑定 FL2VA 校准与 request state + +- 触发：H3 选择 `tea_cache`、修改 extractor、polynomial coefficients、threshold 或 partition。 +- 强制：只允许 `fl2va`/`combined`，Ref2VA-only fail fast；combined 只 hook FL2VA `transformer` 并 + 告警 Ref2VA 不缓存。extractor 必须复刻 `_embed`、packed/SP block kwargs、final row selection 与 + video/audio update masks；缺/多 kwargs、空 blocks、shape/length 不闭合立即失败。runner 在每次 + generation 前 reset module-resident hook state；first step 强制计算，后续按累积距离计算或复用 + residual，目标 hook 没有 last-step 强制计算分支。 +- 强制：H3 coefficients 只由 70 prompts、seed 42–111、256×448、107 frames、50 steps 的 3360 + adjacent pairs 校准；`TeaCacheConfig` 收到 `None` 时模型默认 0.17，自定义 coefficients/正 threshold + 可覆盖。但 public `AsyncOmniEngine._normalize_cache_config` 在省略 cache config 时仍先注入 0.2， + 因此 CLI/public omitted 路径没有获得 0.17；recipe 显式 0.17 不受影响。 +- 禁止：把 FL2VA coefficients 用于 Ref2VA，或宣称 selector 已在 request 层强制互斥。目标 pin 中 + TeaCache 由 runner 持有；`quality=lossless` 只清除 Cache-DiT target、不会卸载 TeaCache， + `quality=high` 还会尝试在同一 transformer 安装 Cache-DiT。修复或拒绝该组合前，TeaCache server + 不得用 request `quality` 承诺 lossless 或双 backend 安全。0.2 单 H100 A/B 虽约 1.20×，LPIPS + 0.3162 是显著视觉差异；0.17 在同 prompt 仅约 3% wall reduction、LPIPS 0.0134，均非通用保证。 + ^[PR #5840] +- 验收：CPU extractor tests 覆盖 native parity、packed kwargs、mask/SP 与错误路径；backend tests + 覆盖 partition/default/override。模型 E2E 仅两 steps 并检查输出 shape，未断言真实 cache hit； + 还必须从 public omitted config 断言最终 H3 threshold，不能只直接构造 backend；覆盖 TeaCache × + omitted/lossless/high，断言只存在一个 hook/backend 或明确拒绝。module state 若可交错还需并发隔离 + 证明。单 H100 online/offline、 + Cache-DiT default/conservative 数据无多 prompt/repeats，不能升级为 gate。 + +## MMH3-2h — distilled sigma schedule 按 partition 所有且以区间计步 + +- 触发：H3 partition `model_index.json` 的 `_minimax_h3.base_schedule`、显式 + `num_inference_steps`、video/audio flow shift 或 combined FL2VA+Ref2VA。 +- 强制：缺少 metadata key 或显式 null 都表示未蒸馏，保持旧 + `num_inference_steps or 50` 的 uniform-point 构造（默认 50 个点实际是 49 个 solver + intervals）；explicit empty list 必须拒绝。调度至少两个有限点，严格 + 从 1.0 递减到 0.0，并逐对验证相邻位置。 +- 强制：5 个 sigma boundaries 只有 4 个 denoise intervals；`num_inference_steps`、 + Cache-DiT quality policy 和请求验证均使用 `len(base_schedule)-1`，solver 仍接收 + 完整 boundary list。用户省略 step 时自动采用 checkpoint 步数；显式值只有 + 精确相等才接受，否则 `OmniClientError`。同一 continuous base positions 分别 + 应用 video/audio shift（默认 12/3），不得改用单模态 integer timesteps。 +- 强制：schedule 从各 partition metadata 分别读取；`t2va`/`fl2va` 选 FL2VA， + `ref2va` 选 Ref2VA。distilled FL2VA 不得拖普通 Ref2VA 进 4-step。类级空 map + 是 partially constructed pipeline 的 legacy fallback，避免 `object.__new__`/dummy fixture + 读未初始化属性。 +- 边界：共享 `DMD2SigmaSchedule` 在 `diffusion/sched` 定义，但此 pin 只有 + H3 消费；现有 `DMD2PipelineMixin` 仍是 scheduler-backed，用 + `DMD2EulerScheduler`/integer `denoising_timesteps`，本次未接入新 utility。不得由 + `[NPU]` title 推导 NPU 实机已验：无 platform/deploy/recipe 改动，PR hardware + 与 branch-head 也留空；附件视频只是 smoke，无数值 teacher/student quality gate。 +- 共享 boundary/interval、metadata null/empty 与 shift 有限性合同见 + [DIFF-4j](../../components/diffusion/sigma-schedules.md#diff-4j-continuous-sigma-boundaries-不得与-scheduler-integer-timesteps-混同)。 +- 验收：shared class 覆盖长度/端点/单调/非有限/shift≤0、缺 key 与空 key； + H3 覆盖双 shift 精确值、combined partition 隔离、4-step 传入 solver/quality、 + matching/mismatched explicit step 和 legacy/partially-constructed fallback。仍缺真实 distilled + checkpoint 的 CUDA/NPU E2E 与 teacher/student 质量阈值。^[PR #5991] + +共享 component quantization、checkpoint mapping 与 quality evidence 见 +[Diffusion rules](../../components/diffusion/rules.md)。 + +部署与容量证据见 [部署与证据规则](rules-deployment.md)；共享 diffusion 缓存机制见 [diffusion 规则](../../components/diffusion/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index ab4a65e12..4c8cbfa20 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -26,12 +26,12 @@ confidence: high | NPU packed varlen、quadratic mask、LaserAttention、prefix K/V | `MMH3-1g` | H3 packed producer → backend capability/metadata → NPU FlashAttention fallback | | FL2VA keyframe、Ref2VA mixed reference/时域限界、shape/output matrix | [MMH3-2a](rules-media.md#mmh3-2a-taskreferenceshape-与多输出必须作为一个输入矩阵维护) | `pipeline_minimax_h3.py` → `reference_video.py` | | media limit、typed/multipart reference、HTTP 400、temp source | [MMH3-2b](rules-media.md#mmh3-2b-media-ingress-在解码前限界request-错误保持-http-400) | `api_server.py` → `serving_video.py` → `reference_video.py` | -| conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | `MMH3-2c` | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | -| modular checkpoint、combined/partition task、两套 DiT、shared component | `MMH3-2d` | model index discovery → startup task selection → task-specific transformer/cache lifecycle | -| request `quality`、lossless/high、dynamic Cache-DiT | `MMH3-2e` | request sampling → quality policy → request Cache-DiT runtime → denoise | -| force-refresh hint、once/repeat、reinstall key | `MMH3-2f` | H3 `extra_args` validation → immutable cache config → installation key/refresh context | -| TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | `MMH3-2g` | custom enabler → H3 extractor → module-resident hook state / per-generation reset | -| distilled、DMD2、`base_schedule`、4-step、sigma boundary | `MMH3-2h` | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | +| conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | [MMH3-2c](rules-cache-task.md#mmh3-2c-conditioned-vae-的固定种子必须按实际设备隔离并恢复) | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | +| modular checkpoint、combined/partition task、两套 DiT、shared component | [MMH3-2d](rules-cache-task.md#mmh3-2d-modular-h3-的-task-selector-必须同步权重、能力与所有-dit-lifecycle) | model index discovery → startup task selection → task-specific transformer/cache lifecycle | +| request `quality`、lossless/high、dynamic Cache-DiT | [MMH3-2e](rules-cache-task.md#mmh3-2e-quality-映射只决定-request-的-cache-dit-目标) | request sampling → quality policy → request Cache-DiT runtime → denoise | +| force-refresh hint、once/repeat、reinstall key | [MMH3-2f](rules-cache-task.md#mmh3-2f-force-refresh-hint-属于-active-profile-identity) | H3 `extra_args` validation → immutable cache config → installation key/refresh context | +| TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | [MMH3-2g](rules-cache-task.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state) | custom enabler → H3 extractor → module-resident hook state / per-generation reset | +| distilled、DMD2、`base_schedule`、4-step、sigma boundary | [MMH3-2h](rules-cache-task.md#mmh3-2h-distilled-sigma-schedule-按-partition-所有且以区间计步) | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | | DLO、TP-local、resident layers、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | @@ -165,145 +165,3 @@ confidence: high tests 再覆盖 malformed fallback 和 Laser math。真实 Ascend parity/内存/延迟证据不能由 CPU mock 替代；同 seed E2E output comparison 与重新抓 memory snapshot 在 PR 中仍列为 follow-up。 ^[PR #5891] ^[PR #5997] - -## MMH3-2c — conditioned VAE 的固定种子必须按实际设备隔离并恢复 - -- 触发：修改 H3 image/video reference 的 VAE encode、固定 keyframe seed、`fork_rng`、设备 - generator 或 accelerator backend 支持。 -- 强制：`encode_image()`/`encode_video()` 在采样前暂时把 VAE 转成 FP32，在 - `torch.random.fork_rng` 同时传 `devices=...` 与 `device_type=parameter.device.type`，并在 - context 内播种 CPU default generator 与 active-device generator；退出时恢复 RNG state， - 在 `finally` 恢复原 dtype，image path 还必须恢复 `parallel_tiling`。这个 seed 是 - conditioned VAE 的固定内部 seed，不能描述成 request seed。 -- 强制：`devices` 只决定保存/恢复哪些 device index，不能代替 `device_type` 选择 RNG module； - 目标 torch 版本省略后者会默认进入 CUDA。NPU 依赖 `torch_npu` 先注册 `torch.npu`，encode - 内的 `self.device_module` 则在同一 active device 上执行 `device(...)` 与 `manual_seed(...)`。 -- 禁止：把 PR 文本中的 CUDA+MUSA allowlist 当成目标实现。目标代码实际以 - `parameter.device.type != "cpu"` 接纳已注册的 accelerator device module，并把真实 type - 交给 `fork_rng`；CPU 的 - `devices=[]` 仍保存/恢复 CPU RNG。实机证据覆盖 CUDA/MUSA，以及 PR #5837 报告的 Ascend - NPU direct fork smoke 与一次双参考图 FL2VA serving 成功，不能外推 XPU 或 ROCm。也不能因 - `fork_rng` 最终恢复 state 就宣称并发安全：context 内仍会暂时改写 process-global CPU/device - generator，重叠调用需要序列化或独立并发证明。 -- 验收：同 seed 的 image/video condition latent 可重复，正常和 encode 异常后 CPU、目标设备 - RNG state、dtype 及 image tiling state 都恢复；CPU 与每个声称支持的 accelerator 分支分别 - 覆盖，并加入重叠调用 fence。PR 中拟议的专用 VAE 单测按 review 被删除，目标 commit 没有 - 新增测试；PR #5837 也只改两处参数，未提交 CPU/NPU 回归测试，其 NPU smoke、serving 结果 - 与 CUDA 不变性说明只能作为外部证据，不能冒充持续回归覆盖。^[PR #5703] ^[PR #5837] - -## MMH3-2d — modular H3 的 task selector 必须同步权重、能力与所有 DiT lifecycle - -- 触发：修改 `modular_model_index.json`、`MiniMaxH3ModularPipeline`、`--task-type`、combined - FL2VA/Ref2VA 服务、`_dit_modules` 或 shared text/VAE components。 -- 强制：模型发现同时识别 `model_index.json` 与 `modular_model_index.json`，registry alias 与 - postprocess 都映射到同一 H3 pipeline。`auto/combined` 从 root 加载 FL2VA `transformer` 与 - Ref2VA `transformers_ref`，但共享 text encoder、video/audio VAE；单 task 只加载所需 DiT。 - request task 必须属于启动时 `supported_tasks`；combined 省略 task 时按无媒体→t2va、image-only - →fl2va、video/audio→ref2va 推断，因此 image-only Ref2VA 必须显式给 task；Ref2VA-only - 省略 task 时保留 implicit ref2va。 -- 强制：modular alias 复制 9-image/mixed-reference admission capability；所有基于 - `_dit_modules` 的 consumer（loader strictness、Cache-DiT enable/refresh/summary、LoRA/offload） - 遍历实际 DiT，不硬编码 `transformer`/`transformer_2`。共享 `--task-type` 放宽后，非 H3 - owner 仍 model-aware 校验；Qwen3-TTS 只接受既有三值。 -- 缺口：目标 pin 的 modular metadata 虽修复 admission 字段，仍遗漏 canonical H3 的 - `attention_mask_free=True`。HF root 的两个 index 都解析成 modular alias，因此默认 - combined 服务在 Blackwell 不会自动选 TRTLLM，与 recipe 声明冲突；alias 必须校验全部能力 - 字段 parity，不能只复制 review 点名的字段。 -- 禁止：从 combined 注册推断双 DiT 性能已验证；recipe 最终只描述配置，没有 combined - warm latency/throughput/output-parity qualification。也不能用单 DiT Cache-DiT 测试证明 - `transformers_ref` lifecycle。 -- 验收：覆盖 local/Hub index discovery、snapshot allow-pattern、combined/单分区初始化、缺 - partition、task default/membership、modular admission、两 DiT cache lifecycle 与非法 TTS - selector。PR body 的顺序 T2VA→Ref2VA 视频使用 `duration=2.0`，低于目标 pin 已生效的 4 秒 - 下限，因此不能证明 final merge 的 combined 路径；148.27/158.67 GiB combined 与 86.27 GiB - single-load 等数字来自缺硬件/协议/重复次数的 issue comment，不能作为容量保证。^[PR #5720] - -## MMH3-2e — quality 映射只决定 request 的 Cache-DiT 目标 - -- 触发：修改公共 `quality` 值、H3 quality policy、startup cache adoption 或 denoise 前 prepare。 -- 强制：公共层只接受 `None`、`lossless`、`high`，并保留 omitted 与 explicit lossless 的区别。 - 对 request-scoped Cache-DiT，`lossless` 产生空 target；`high` 总是选择 model-owned conservative - profile，即使启动时没有 cache backend；omitted 仅在启动配置为 Cache-DiT 时恢复 server generic - profile，否则不安装 Cache-DiT。 - pipeline 必须在参数/任务/step 解析完成后、真实 denoise 紧邻之前 apply plan。 -- 禁止：沿用 PR 早期描述，把无 startup cache 的 `high` 拒绝；让 unsupported quality 静默落入 - lossless；在 request 未显式提供 quality 时由 sync/streaming serving 覆盖 model default。recipe - 把同一组 quality 数字标为 4×H200，而 PR evidence 把它归于 4×L20X SP4；hardware provenance - 未统一前不得引用该表作为任一硬件的可靠 benchmark，更不能外推通用 speedup/quality。 -- 验收：offline、chat、sync video、streaming video 都覆盖 omitted/lossless/high/非法值及显式字段 - 传播；startup cache on/off × 三种 intent 验证 exact installation key，并断言 prepare 先于 diffuse。 - 当前测试广泛覆盖 validation/routing/policy 和 mock 顺序，但真实 Cache-DiT transition 的并发、取消、 - failure rollback 仍缺证据；独立 reviewer 的单卡 B300 观察也显示 `high` 的质量/延迟与四卡表明显 - 不同，只能支持 topology-dependent 边界。共享状态机与 batch 合同见 - [DIFF-2i–2j](../../components/diffusion/rules-component-lifecycle.md)。^[PR #5853] - -## MMH3-2f — force-refresh hint 属于 active profile identity - -- 触发：H3 request `extra_args` 使用 `force_refresh_step_hint` 或 `force_refresh_step_policy`。 -- 强制：hint 是 1-based positive integer 且不超过 `num_inference_steps`，policy 只接受 once/repeat， - 省略 policy 默认 once；没有 active cache target 时两者都拒绝。用 dataclass replace 生成 request-local - config，不修改 startup generic config；installation key 必须包含 hint+policy，因为 Cache-DiT 的 - incremental refresh 把 `None` 解释为保留旧 hint，变更或移除 hint 必须 reinstall hooks。 -- 禁止：接受 bool 作为整数；只给 policy；跨 request 原地修改共享 config；same-key repeated request - 忘记重置 once hint，导致它只在首个 request 生效。 -- 验收：边界 1/steps、0/越界/bool/非法 policy、无 target、hint add/change/remove、once/repeat 和 - repeated same-key request；断言每次 refresh 重建 hint context、generic config 不变，key 变化发生 - disable→enable。当前覆盖为 mock/config 合同，未证明不同 topology 下 hint 对质量/命中率的效果。 - ^[PR #5853] - -## MMH3-2g — H3 TeaCache 只绑定 FL2VA 校准与 request state - -- 触发：H3 选择 `tea_cache`、修改 extractor、polynomial coefficients、threshold 或 partition。 -- 强制：只允许 `fl2va`/`combined`，Ref2VA-only fail fast；combined 只 hook FL2VA `transformer` 并 - 告警 Ref2VA 不缓存。extractor 必须复刻 `_embed`、packed/SP block kwargs、final row selection 与 - video/audio update masks；缺/多 kwargs、空 blocks、shape/length 不闭合立即失败。runner 在每次 - generation 前 reset module-resident hook state；first step 强制计算，后续按累积距离计算或复用 - residual，目标 hook 没有 last-step 强制计算分支。 -- 强制：H3 coefficients 只由 70 prompts、seed 42–111、256×448、107 frames、50 steps 的 3360 - adjacent pairs 校准；`TeaCacheConfig` 收到 `None` 时模型默认 0.17，自定义 coefficients/正 threshold - 可覆盖。但 public `AsyncOmniEngine._normalize_cache_config` 在省略 cache config 时仍先注入 0.2， - 因此 CLI/public omitted 路径没有获得 0.17；recipe 显式 0.17 不受影响。 -- 禁止：把 FL2VA coefficients 用于 Ref2VA，或宣称 selector 已在 request 层强制互斥。目标 pin 中 - TeaCache 由 runner 持有；`quality=lossless` 只清除 Cache-DiT target、不会卸载 TeaCache， - `quality=high` 还会尝试在同一 transformer 安装 Cache-DiT。修复或拒绝该组合前，TeaCache server - 不得用 request `quality` 承诺 lossless 或双 backend 安全。0.2 单 H100 A/B 虽约 1.20×，LPIPS - 0.3162 是显著视觉差异；0.17 在同 prompt 仅约 3% wall reduction、LPIPS 0.0134，均非通用保证。 - ^[PR #5840] -- 验收：CPU extractor tests 覆盖 native parity、packed kwargs、mask/SP 与错误路径；backend tests - 覆盖 partition/default/override。模型 E2E 仅两 steps 并检查输出 shape，未断言真实 cache hit； - 还必须从 public omitted config 断言最终 H3 threshold，不能只直接构造 backend；覆盖 TeaCache × - omitted/lossless/high，断言只存在一个 hook/backend 或明确拒绝。module state 若可交错还需并发隔离 - 证明。单 H100 online/offline、 - Cache-DiT default/conservative 数据无多 prompt/repeats，不能升级为 gate。 - -## MMH3-2h — distilled sigma schedule 按 partition 所有且以区间计步 - -- 触发：H3 partition `model_index.json` 的 `_minimax_h3.base_schedule`、显式 - `num_inference_steps`、video/audio flow shift 或 combined FL2VA+Ref2VA。 -- 强制：缺少 metadata key 或显式 null 都表示未蒸馏，保持旧 - `num_inference_steps or 50` 的 uniform-point 构造（默认 50 个点实际是 49 个 solver - intervals）；explicit empty list 必须拒绝。调度至少两个有限点，严格 - 从 1.0 递减到 0.0，并逐对验证相邻位置。 -- 强制：5 个 sigma boundaries 只有 4 个 denoise intervals；`num_inference_steps`、 - Cache-DiT quality policy 和请求验证均使用 `len(base_schedule)-1`，solver 仍接收 - 完整 boundary list。用户省略 step 时自动采用 checkpoint 步数；显式值只有 - 精确相等才接受，否则 `OmniClientError`。同一 continuous base positions 分别 - 应用 video/audio shift（默认 12/3），不得改用单模态 integer timesteps。 -- 强制：schedule 从各 partition metadata 分别读取；`t2va`/`fl2va` 选 FL2VA， - `ref2va` 选 Ref2VA。distilled FL2VA 不得拖普通 Ref2VA 进 4-step。类级空 map - 是 partially constructed pipeline 的 legacy fallback，避免 `object.__new__`/dummy fixture - 读未初始化属性。 -- 边界：共享 `DMD2SigmaSchedule` 在 `diffusion/sched` 定义，但此 pin 只有 - H3 消费；现有 `DMD2PipelineMixin` 仍是 scheduler-backed，用 - `DMD2EulerScheduler`/integer `denoising_timesteps`，本次未接入新 utility。不得由 - `[NPU]` title 推导 NPU 实机已验：无 platform/deploy/recipe 改动，PR hardware - 与 branch-head 也留空；附件视频只是 smoke，无数值 teacher/student quality gate。 -- 共享 boundary/interval、metadata null/empty 与 shift 有限性合同见 - [DIFF-4j](../../components/diffusion/sigma-schedules.md#diff-4j-continuous-sigma-boundaries-不得与-scheduler-integer-timesteps-混同)。 -- 验收：shared class 覆盖长度/端点/单调/非有限/shift≤0、缺 key 与空 key； - H3 覆盖双 shift 精确值、combined partition 隔离、4-step 传入 solver/quality、 - matching/mismatched explicit step 和 legacy/partially-constructed fallback。仍缺真实 distilled - checkpoint 的 CUDA/NPU E2E 与 teacher/student 质量阈值。^[PR #5991] - -共享 component quantization、checkpoint mapping 与 quality evidence 见 -[Diffusion rules](../../components/diffusion/rules.md)。 From 1059e4aa5f07c663e443f46632134981bbf09e02 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 03:56:03 +0800 Subject: [PATCH 131/465] knowledge(vllm-omni): sync 9a594750 (PR #6129) Upstream-Commit: 9a59475015102608dc8655c967e817a66581e8ef Upstream-PR: #6129 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ecd8db870..aab21d552 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 72a02b492a6957acc4659fde09be85c00ad0f90b + previous_audited_sha: 12f81681793441632ffb7d68b200b9d61821e1f8 audited_ref: main - audited_sha: 12f81681793441632ffb7d68b200b9d61821e1f8 + audited_sha: 9a59475015102608dc8655c967e817a66581e8ef # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c63ea1a41..07ab82b21 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -12f81681793441632ffb7d68b200b9d61821e1f8`. The canonical machine baseline +9a59475015102608dc8655c967e817a66581e8ef`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b3a2fa14d..16f263a28 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 12f81681` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9a594750` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 12a0db955..1452a2a59 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 12f81681` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9a594750` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c786dd0cd..651f1857c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 12f81681` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9a594750` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6284292a6..e44ba228c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 12f81681` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9a594750` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c572453a6..0183c5d5a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 12f81681` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9a594750` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f307bc64a..8bb181966 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 12f81681` 验证存在 +- 源码校验：以上路径均已在 `main @ 9a594750` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c0aa6ecc1..9c74473f0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 12f81681`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9a594750`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `72a02b49` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `12f81681` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 72a02b49 \ - --to 12f81681 \ + --from 12f81681 \ + --to 9a594750 \ --repo \ --mode report-only ``` From c22ac8d203ce06da6cbad417bd9336471d03442b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 03:56:34 +0800 Subject: [PATCH 132/465] knowledge(vllm-omni): sync c57178c0 (PR #6065) Upstream-Commit: c57178c0e18df7b622fe6e3ee8357495305876b5 Upstream-PR: #6065 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index aab21d552..3d8501c4e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 12f81681793441632ffb7d68b200b9d61821e1f8 + previous_audited_sha: 9a59475015102608dc8655c967e817a66581e8ef audited_ref: main - audited_sha: 9a59475015102608dc8655c967e817a66581e8ef + audited_sha: c57178c0e18df7b622fe6e3ee8357495305876b5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 07ab82b21..c69f41a38 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -9a59475015102608dc8655c967e817a66581e8ef`. The canonical machine baseline +c57178c0e18df7b622fe6e3ee8357495305876b5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 16f263a28..2f20b0e30 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9a594750` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c57178c0` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1452a2a59..c68c6039b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9a594750` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c57178c0` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 651f1857c..27dedf357 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9a594750` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c57178c0` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e44ba228c..4d38a5d2a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9a594750` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c57178c0` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0183c5d5a..8407338aa 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9a594750` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c57178c0` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8bb181966..28fe7b182 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9a594750` 验证存在 +- 源码校验：以上路径均已在 `main @ c57178c0` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9c74473f0..8ddd4a6ec 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9a594750`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c57178c0`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `12f81681` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `9a594750` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 12f81681 \ - --to 9a594750 \ + --from 9a594750 \ + --to c57178c0 \ --repo \ --mode report-only ``` From 62fbfd37526532045e9badf69271381b163dfc4e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 03:58:14 +0800 Subject: [PATCH 133/465] knowledge(vllm-omni): sync 8b8d3f56 (PR #6076) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 8b8d3f56db4a1e39c635db73fec96112918d8083 Upstream-PR: #6076 Knowledge-Delta: 新增 EXEC-6b：model_extras 必须集中声明模型专有请求参数和浮点输出张量范围，导出侧按声明统一归一化。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3d8501c4e..f6500e060 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9a59475015102608dc8655c967e817a66581e8ef + previous_audited_sha: c57178c0e18df7b622fe6e3ee8357495305876b5 audited_ref: main - audited_sha: c57178c0e18df7b622fe6e3ee8357495305876b5 + audited_sha: 8b8d3f56db4a1e39c635db73fec96112918d8083 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c69f41a38..0a3d37717 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c57178c0e18df7b622fe6e3ee8357495305876b5`. The canonical machine baseline +8b8d3f56db4a1e39c635db73fec96112918d8083`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2f20b0e30..08f3c7465 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c57178c0` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8b8d3f56` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c68c6039b..05c57e388 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c57178c0` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8b8d3f56` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 27dedf357..efba70bf2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c57178c0` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8b8d3f56` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4d38a5d2a..dce19d5c0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c57178c0` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8b8d3f56` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 06ad2d9f9..66d9fe255 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076"] --- # Model Executor 规则 @@ -193,6 +193,13 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 覆盖 Bagel、MammothModa2、Ming、VACE custom path及 unknown identity path；Cosmos3/LingBot pipeline tests继续拥有模型专属 validation。^[PR #6049] +### EXEC-6b — model_extras 必须声明模型专有参数与输出张量范围 + +- 触发：新增或修改 `model_extras` 中模型专有请求参数、输出张量范围，或共享视频导出对不同 pipeline 输出合同的处理时。 +- 强制：按解析后的 `model_class_name` 在 registry 中声明 `extra_body_params` 与 `output_tensor_range`，通过公开 accessor 提供；消费者按整个视频的声明合同统一转换浮点张量，未声明模型保持 `negative_one_to_one` 默认行为。 +- 禁止：把模型专有选项散落到共享 runner 的通用参数或绕过 registry；按每帧当前最小值推断范围；把已是 `[0, 1]` 的模型输出再次按 `[-1, 1]` 映射。 +- 验收：registry 测试断言目标模型的 extra-body 参数和 `zero_to_one` 范围、普通 pipeline 的默认范围；覆盖单帧与 list-valued 视频的明确范围、混合正负值的统一转换，以及最终视频导出结果。 ^[PR #6076] + ### EXEC-7a — Omni 输出必须是扁平的 RequestOutput 子类 - 触发：修改 `OmniRequestOutput`、stage output wrapping、`omni.generate()` 返回对象或其 serving/offline 消费者。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8407338aa..a1709c6c4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c57178c0` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8b8d3f56` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 28fe7b182..5c260874f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c57178c0` 验证存在 +- 源码校验：以上路径均已在 `main @ 8b8d3f56` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8ddd4a6ec..c5be27051 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c57178c0`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8b8d3f56`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `9a594750` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `c57178c0` 本轮新增 model_extras 对模型专有请求参数和输出张量范围的声明规则；基准、测试、文档及共享离线 runner 的迁移不应据此推断为 model-executor 规则，在线 serving 合同也未发生变化。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 9a594750 \ - --to c57178c0 \ + --from c57178c0 \ + --to 8b8d3f56 \ --repo \ --mode report-only ``` From e5308bb8aad97c090bb6e7467f86703ffa3bc705 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:01:51 +0800 Subject: [PATCH 134/465] knowledge(vllm-omni): sync 877beeaa (PR #6082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 877beeaa4957e8caab0daad31930cb274cb601cb Upstream-PR: #6082 Knowledge-Delta: 新增 MiniCPM-o 4.5 CFM DiT CUDA Graph 的 deploy 配置一致性、按 shape 缓存、eager fallback 与数值 parity 门禁。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 6 +++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md | 2 +- knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md | 8 ++++++-- 12 files changed, 25 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f6500e060..d6443e49e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c57178c0e18df7b622fe6e3ee8357495305876b5 + previous_audited_sha: 8b8d3f56db4a1e39c635db73fec96112918d8083 audited_ref: main - audited_sha: 8b8d3f56db4a1e39c635db73fec96112918d8083 + audited_sha: 877beeaa4957e8caab0daad31930cb274cb601cb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0a3d37717..c5c78ef50 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8b8d3f56db4a1e39c635db73fec96112918d8083`. The canonical machine baseline +877beeaa4957e8caab0daad31930cb274cb601cb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 08f3c7465..dd3b0180f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8b8d3f56` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 877beeaa` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index be54209b8..9b75255fb 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082"] --- # vLLM-Omni 配置开发门禁 @@ -204,3 +204,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 验收：用 bare `config.json` 和三个 architecture 分别验证 model type、pipeline、stage config 与默认 deploy 名称；sync/async 配置只切换 processor wiring，并保持三 stage 顺序与 stage architecture 一致。 ^[PR #5842] +### CONF-5e — MiniCPM-o 部署 profile 必须一致传递 CFM Graph 开关 + +- 触发：修改 MiniCPM-o 4.5 bundled deploy YAML 或 Code2Wav connector extra 中的 CFM CUDA Graph 开关、缓存上限或配置传递。\n- 强制：`minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml`、`minicpmo_4_5_3gpu.yaml` 和 `minicpmo_4_5_8x4090.yaml` 必须一致声明 `enable_cfm_graph` 与 `cfm_max_graphs`；当前默认值为 `true` 与 `32`，并传入 `MiniCPMO45Code2Wav` 的 `_cfm_graph_config` 和 `BatchedToken2Wav`。非 CUDA 设备仍必须回退 eager。\n- 禁止：只更新一个 deploy profile、依赖未记录的默认值，或把配置存在误认为 graph 已在非 CUDA 环境生效；不得把模型专有开关扩展成所有模型的通用配置合同。\n- 验收：解析四份 deploy 配置并断言开关和上限一致，追踪非默认值到 `_cfm_graph_config` 与 `BatchedToken2Wav`；覆盖开关关闭和非 CUDA fallback，并确认 stage topology 未发生额外变化。\n^[PR #6082] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 05c57e388..89b579c12 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8b8d3f56` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 877beeaa` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index efba70bf2..fd91763a7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8b8d3f56` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 877beeaa` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index dce19d5c0..5ac898db6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8b8d3f56` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 877beeaa` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a1709c6c4..421b5b95e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8b8d3f56` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 877beeaa` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5c260874f..60fee3b05 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8b8d3f56` 验证存在 +- 源码校验：以上路径均已在 `main @ 877beeaa` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c5be27051..4853a7462 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8b8d3f56`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 877beeaa`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `c57178c0` 本轮新增 model_extras 对模型专有请求参数和输出张量范围的声明规则；基准、测试、文档及共享离线 runner 的迁移不应据此推断为 model-executor 规则，在线 serving 合同也未发生变化。 +本次相对 `8b8d3f56` 本轮将 PR #6082 的 MiniCPM-o 4.5 CFM DiT CUDA Graph 门禁写入配置与模型规则，并补充来源；不应据此推断通用 CUDA Graph、其他模型或环境绑定的吞吐/WER 数据已获得验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from c57178c0 \ - --to 8b8d3f56 \ + --from 8b8d3f56 \ + --to 877beeaa \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 3ba8209e5..5ae8e3204 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py] +sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 99fedc17d..17880b580 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -1,10 +1,10 @@ --- title: "MiniCPM-o 4.5 规则" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082"] confidence: high --- @@ -89,6 +89,10 @@ confidence: high 250→43 ms/request，只绑定部分 graph、 单 prompt/commit `8fa28d88`，无 repeats/端到端质量，不能泛化为稳定 speedup。^[PR #5869] +## MCPMO-1e — CFM DiT CUDA Graph 必须按 shape 捕获并保持 eager parity + +- 触发：修改 MiniCPM-o 4.5 Code2Wav CFM DiT estimator 的 CUDA Graph 开关、`blocks_forward_chunk` graph target、按 shape 捕获/重放、cache eviction 或 eager fallback。\n- 强制：仅在 CUDA 且显式启用时捕获 `estimator.blocks_forward_chunk`；`t_embedder`、`torch.cat`、`expand`、buffer 准备和 10-step Euler loop 保持 eager。graph key 必须包含所有输入的 shape、dtype 和 device，并使用有界 `cfm_max_graphs` LRU；`cnn_cache=None` 与 `att_cache=None` 转为 graph 可接受的零值时，必须保持与 eager `[None] * depth` 的首 chunk 数值语义，重放结果和 cache 输出需 clone。\n- 禁止：捕获整个 `_estimator_step` 或手写替代 graph target；使用无界 shape cache；在 nested capture 中重放；捕获失败后伪装成功；把 CFM graph 的配置、性能或 parity 结论外推到 HiFT、encoder、TRT 或其他模型。\n- 验收：CUDA 测试覆盖 uncached/cached shape 的 graph/eager 数值 parity、`None` cache parity、lazy capture、capture failure eager fallback 和 LRU eviction；同时覆盖 active capture/non-CUDA fallback，并确认 deploy 配置值实际到达 estimator。\n^[PR #6082] + ## MCPMO-2a — registry 使用 4.5 config/version predicate - 触发：pipeline auto-detection 看到通用 `MiniCPMO` architecture。 From 1d35ea410c8ceeea23e47929fe9905894a8f1cca Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:02:21 +0800 Subject: [PATCH 135/465] knowledge(vllm-omni): sync d77a35a3 (PR #6085) Upstream-Commit: d77a35a32b25d1460ac5c72f66f4d9fd554399a2 Upstream-PR: #6085 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d6443e49e..89456458d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8b8d3f56db4a1e39c635db73fec96112918d8083 + previous_audited_sha: 877beeaa4957e8caab0daad31930cb274cb601cb audited_ref: main - audited_sha: 877beeaa4957e8caab0daad31930cb274cb601cb + audited_sha: d77a35a32b25d1460ac5c72f66f4d9fd554399a2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c5c78ef50..e31e30195 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -877beeaa4957e8caab0daad31930cb274cb601cb`. The canonical machine baseline +d77a35a32b25d1460ac5c72f66f4d9fd554399a2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index dd3b0180f..aa25b2579 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 877beeaa` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d77a35a3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 89b579c12..b69722f82 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 877beeaa` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d77a35a3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index fd91763a7..d45bc851a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 877beeaa` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d77a35a3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5ac898db6..c267e00ac 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 877beeaa` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d77a35a3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 421b5b95e..ab8eb8316 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 877beeaa` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d77a35a3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 60fee3b05..2bab0477d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 877beeaa` 验证存在 +- 源码校验：以上路径均已在 `main @ d77a35a3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4853a7462..1013e765f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 877beeaa`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d77a35a3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `8b8d3f56` 本轮将 PR #6082 的 MiniCPM-o 4.5 CFM DiT CUDA Graph 门禁写入配置与模型规则，并补充来源；不应据此推断通用 CUDA Graph、其他模型或环境绑定的吞吐/WER 数据已获得验证。 +本次相对 `877beeaa` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 8b8d3f56 \ - --to 877beeaa \ + --from 877beeaa \ + --to d77a35a3 \ --repo \ --mode report-only ``` From c1a62cfd4af12632b602b2fdf18c571cf4f88742 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:04:47 +0800 Subject: [PATCH 136/465] knowledge(vllm-omni): sync c98517ce (PR #4755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: c98517ce7fab78221e7806b56dd862a5fb69eae4 Upstream-PR: #4755 Knowledge-Delta: 新增 diffusion per-step timing metadata 保留与 serving result 去重累计规则；observability、文档和测试 owner 的变更按页面范围丢弃。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/rules.md | 12 ++++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 31 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 89456458d..3631985bc 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 877beeaa4957e8caab0daad31930cb274cb601cb + previous_audited_sha: d77a35a32b25d1460ac5c72f66f4d9fd554399a2 audited_ref: main - audited_sha: d77a35a32b25d1460ac5c72f66f4d9fd554399a2 + audited_sha: c98517ce7fab78221e7806b56dd862a5fb69eae4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e31e30195..766f0061f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d77a35a32b25d1460ac5c72f66f4d9fd554399a2`. The canonical machine baseline +c98517ce7fab78221e7806b56dd862a5fb69eae4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index aa25b2579..fb7bddccc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d77a35a3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c98517ce` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b69722f82..2d242b45b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d77a35a3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c98517ce` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 6ce2d6d29..7d1b29c14 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -1,10 +1,10 @@ --- title: "Diffusion 共享规则" created: 2026-07-20 -updated: 2026-09-03 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755"] confidence: high --- @@ -214,5 +214,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 验收：运行测试文件中的 exact case，并在规则/配置旁保留最短资源原因；对称 baseline 证明质量差异来自目标量化变量。低 steps smoke 只能证明 runtime compatibility，不得声称 BF16 quality parity。 ^[PR #5136] ^[PR #6279] + +### DIFF-5a — diffusion metrics 的 per-step 计算必须保留请求步数 + +- 触发：修改 diffusion output formatter、sampling params metadata 或 diffusion metrics accumulation，使请求级 DiT execution time 需要按 denoising step 归一化。 +- 强制：`format_diffusion_outputs()` 必须保留 `sampling_params.num_inference_steps` 并传入 stage stats；aggregator 保留该 scalar，仅在 execution time 与正步数同时存在时按 `exec_time / num_inference_steps` 观测 per-step metric。 +- 禁止：丢弃请求步数、用 stage 总生成时间替代 DiT execution time，或把跨多个结果的 scalar 累加后再计算 per-step 值。 +- 验收：覆盖完整 metadata、缺失/`None`/零步数、正步数、重复结果与零 execution time；分别断言 per-step 值精确、无效步数跳过且 scalar 不累加。^[PR #4755] + 相关执行流见 [Diffusion architecture](architecture.md)；benchmark 证据合同见 [performance evidence](../../benchmark/guides/performance-evidence.md)。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d45bc851a..e11c095bb 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d77a35a3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c98517ce` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c267e00ac..9ac22df97 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d77a35a3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c98517ce` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ab8eb8316..9368194d7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d77a35a3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c98517ce` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2bab0477d..b4b20d0da 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d77a35a3` 验证存在 +- 源码校验：以上路径均已在 `main @ c98517ce` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 842d5bea6..6112d20ee 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755"] confidence: high --- @@ -137,6 +137,13 @@ confidence: high - 验收：同进程连续创建/销毁两次 engine，无 duplicate-timeseries 错误且 Omni family 仍可采集。 ^[PR #3576] +## SERV-2c — stage metrics 只能在 result message 首次消费时累计 + +- 触发：修改 `OmniBase` 的 result message 去重、`accumulate_diffusion_metrics`、`on_stage_metrics` 或重试/流式结果处理。 +- 强制：以每个请求的 `msg_id = id(result)` consumed 集合作为唯一门禁；仅在消息首次未消费时累计 diffusion metrics、处理 stage metrics，完成两项操作后再记录该消息已消费。 +- 禁止：在轮询重试、重复回调或消息标记后再次累计同一 result；把同一消息误当作新的 denoising step，或绕过现有 consumed 生命周期清理。 +- 验收：同一 result 重复处理只产生一次累计，两个 distinct result 各自产生一次；覆盖异常、重试与请求清理，确认 consumed 状态不会导致重复累计或遗留。^[PR #4755] + ## 缓存 readiness 与失败隔离 ### SERV-3a — readiness 必须表达 artifact 能力而非仅表达 identity diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1013e765f..5fd435dc4 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d77a35a3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c98517ce`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `877beeaa` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `d77a35a3` 本轮新增 diffusion per-step timing metadata 保留与 serving stage-result 去重规则。未写入 observability、文档或测试 owner；因此不能据此推断完整指标族、bucket 配置或所有模型/部署已验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 877beeaa \ - --to d77a35a3 \ + --from d77a35a3 \ + --to c98517ce \ --repo \ --mode report-only ``` From 705062c3886a4c54ce2e1c523670cb1abaddec72 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:05:52 +0800 Subject: [PATCH 137/465] knowledge(vllm-omni): sync bbb709cf (PR #6156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: bbb709cfb56358500cbd89a9cca0b3a24e66dbe5 Upstream-PR: #6156 Knowledge-Delta: 新增 configuration 的 deploy-sampling 审查组，记录 MOSS-TTS-Local 必须采用官方采样参数。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 8 +++++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 20 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3631985bc..71242082b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d77a35a32b25d1460ac5c72f66f4d9fd554399a2 + previous_audited_sha: c98517ce7fab78221e7806b56dd862a5fb69eae4 audited_ref: main - audited_sha: c98517ce7fab78221e7806b56dd862a5fb69eae4 + audited_sha: bbb709cfb56358500cbd89a9cca0b3a24e66dbe5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 766f0061f..090d0d86e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c98517ce7fab78221e7806b56dd862a5fb69eae4`. The canonical machine baseline +bbb709cfb56358500cbd89a9cca0b3a24e66dbe5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fb7bddccc..6b527effc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c98517ce` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bbb709cf` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 9b75255fb..b9a6b9d62 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156"] --- # vLLM-Omni 配置开发门禁 @@ -208,3 +208,9 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 触发：修改 MiniCPM-o 4.5 bundled deploy YAML 或 Code2Wav connector extra 中的 CFM CUDA Graph 开关、缓存上限或配置传递。\n- 强制：`minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml`、`minicpmo_4_5_3gpu.yaml` 和 `minicpmo_4_5_8x4090.yaml` 必须一致声明 `enable_cfm_graph` 与 `cfm_max_graphs`；当前默认值为 `true` 与 `32`，并传入 `MiniCPMO45Code2Wav` 的 `_cfm_graph_config` 和 `BatchedToken2Wav`。非 CUDA 设备仍必须回退 eager。\n- 禁止：只更新一个 deploy profile、依赖未记录的默认值，或把配置存在误认为 graph 已在非 CUDA 环境生效；不得把模型专有开关扩展成所有模型的通用配置合同。\n- 验收：解析四份 deploy 配置并断言开关和上限一致，追踪非默认值到 `_cfm_graph_config` 与 `BatchedToken2Wav`；覆盖开关关闭和非 CUDA fallback，并确认 stage topology 未发生额外变化。\n^[PR #6082] +### CONF-6a — MOSS-TTS-Local 部署必须采用官方采样参数 + +- 触发：修改 MOSS-TTS-Local 的 deploy YAML 或其默认采样参数时。 +- 强制：采用 MOSS-TTS-Local 官方示例参数：`temperature=1.7`、`top_p=0.8`、`top_k=25`、`repetition_penalty=1.0`；如需变更，必须有模型级行为证据支持。 +- 禁止：沿用通用默认采样值，或仅凭经验调整敏感采样参数而不核对官方示例。 +- 验收：解析 `vllm_omni/deploy/moss_tts_local.yaml`，断言上述参数与官方示例一致，并确认最终 stage 配置实际读取这些值。 ^[PR #6156] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2d242b45b..d413b881d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c98517ce` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bbb709cf` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e11c095bb..85bbd4e3c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c98517ce` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bbb709cf` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9ac22df97..b1518f63d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c98517ce` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bbb709cf` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9368194d7..86cde4939 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c98517ce` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bbb709cf` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b4b20d0da..2ec093d2f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c98517ce` 验证存在 +- 源码校验：以上路径均已在 `main @ bbb709cf` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5fd435dc4..b2a178737 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c98517ce`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bbb709cf`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `d77a35a3` 本轮新增 diffusion per-step timing metadata 保留与 serving stage-result 去重规则。未写入 observability、文档或测试 owner；因此不能据此推断完整指标族、bucket 配置或所有模型/部署已验证。 +本次相对 `c98517ce` 本轮为 configuration 规则页新增 MOSS-TTS-Local 官方采样参数门禁；仅表示该部署配置的采样参数必须遵循官方示例，不应推断为所有模型或所有部署的通用采样策略。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from d77a35a3 \ - --to c98517ce \ + --from c98517ce \ + --to bbb709cf \ --repo \ --mode report-only ``` From cdadeb12b46ad759bfc068244a046e4641f8dcc7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:06:23 +0800 Subject: [PATCH 138/465] knowledge(vllm-omni): sync bbe6ccc5 (PR #6141) Upstream-Commit: bbe6ccc512a404a2df8c977ea29003002f2683e8 Upstream-PR: #6141 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 71242082b..077a1a637 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c98517ce7fab78221e7806b56dd862a5fb69eae4 + previous_audited_sha: bbb709cfb56358500cbd89a9cca0b3a24e66dbe5 audited_ref: main - audited_sha: bbb709cfb56358500cbd89a9cca0b3a24e66dbe5 + audited_sha: bbe6ccc512a404a2df8c977ea29003002f2683e8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 090d0d86e..9c5b5ce6f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bbb709cfb56358500cbd89a9cca0b3a24e66dbe5`. The canonical machine baseline +bbe6ccc512a404a2df8c977ea29003002f2683e8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6b527effc..b65eb308c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bbb709cf` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bbe6ccc5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d413b881d..61a328a02 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bbb709cf` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bbe6ccc5` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 85bbd4e3c..61d95c105 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bbb709cf` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bbe6ccc5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b1518f63d..1db089327 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bbb709cf` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bbe6ccc5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 86cde4939..0dcb5ba2a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bbb709cf` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bbe6ccc5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2ec093d2f..38297eb99 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bbb709cf` 验证存在 +- 源码校验：以上路径均已在 `main @ bbe6ccc5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b2a178737..bf491f8cf 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bbb709cf`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bbe6ccc5`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `c98517ce` 本轮为 configuration 规则页新增 MOSS-TTS-Local 官方采样参数门禁；仅表示该部署配置的采样参数必须遵循官方示例，不应推断为所有模型或所有部署的通用采样策略。 +本次相对 `bbb709cf` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from c98517ce \ - --to bbb709cf \ + --from bbb709cf \ + --to bbe6ccc5 \ --repo \ --mode report-only ``` From 7eb3c4dbe77a3e4ab2d148d90cf6a94c34628f62 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:08:22 +0800 Subject: [PATCH 139/465] knowledge(vllm-omni): sync f41451f6 (PR #6096) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: f41451f64c8b2fc20f8296f4c2a12c7d3bb16181 Upstream-PR: #6096 Knowledge-Delta: 新增 NPU KV connector finalize 的 PP 条件与 runtime mode 有效性校验规则；CI 镜像升级和已有 dummy-run randomize 合同不重复沉淀。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/model-executor/rules.md | 16 +++++++++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 28 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 077a1a637..2b8ea6247 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bbb709cfb56358500cbd89a9cca0b3a24e66dbe5 + previous_audited_sha: bbe6ccc512a404a2df8c977ea29003002f2683e8 audited_ref: main - audited_sha: bbe6ccc512a404a2df8c977ea29003002f2683e8 + audited_sha: f41451f64c8b2fc20f8296f4c2a12c7d3bb16181 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9c5b5ce6f..614baeaf4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bbe6ccc512a404a2df8c977ea29003002f2683e8`. The canonical machine baseline +f41451f64c8b2fc20f8296f4c2a12c7d3bb16181`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b65eb308c..6dfd2ea7d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bbe6ccc5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f41451f6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 61a328a02..2df26e085 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bbe6ccc5` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f41451f6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 61d95c105..8f3a0b549 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bbe6ccc5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f41451f6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1db089327..2db0abe02 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bbe6ccc5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f41451f6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 66d9fe255..91ccce2d6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096"] --- # Model Executor 规则 @@ -148,6 +148,13 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 ## 跨 stage bridge 与 batch 合同 +### EXEC-5b — NPU KV connector finalize 必须按 PP 所有权延迟 + +- 触发：NPU AR 或 generation runner 在 speculative decoding、pipeline parallel 或 `broadcast_pp_output` 场景调用 `maybe_get_kv_connector_output`。 +- 强制：仅当 `self.speculative_config is not None` 且当前 rank 是 `get_pp_group().is_last_rank` 或 `self.broadcast_pp_output` 为真时设置 `defer_finalize=True`；AR 与 generation runner 必须保持相同条件。 +- 禁止：仅依据 speculative decoding 开启状态就在所有 PP rank 延迟 finalize，或继续使用不区分 PP 所有权的 `clear_kv_metadata` 反向条件。 +- 验收：覆盖无 speculative config、speculative config、last rank、非 last rank 和 `broadcast_pp_output` 的组合，断言两个 runner 传入 connector 的 `defer_finalize` 值及最终 finalize 时机一致。 ^[PR #6096] + ### EXEC-2a — loader 的 dtype 与 config 获取必须显式、最小化 - 触发：模型 loader 构造 text encoder、VAE、transformer 或只读取 checkpoint config。 @@ -220,3 +227,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：将循环不变的文本 embedding、`audio_mask_3d`、RoPE 和 additive attention mask 在循环外缓存；批量取得 `c_lens` 并只在实际消费的 logits slice 上转为 fp32；CUDA graph 与 eager 使用相同的 float mask 语义，bool mask 必须将 True 映射为 `0.0`、False 映射为 `-inf`，已有 float mask 必须原样保留；TF32 只能通过显式配置 opt-in，并在 graph capture 前启用。 - 禁止：每步重复计算固定输入或执行逐请求 D2H `.item()`；对 float mask 使用布尔反转或把 bool 直接隐式转换进 float buffer；让 graph capture 与 replay 使用不同 mask dtype；默认开启或宣称 TF32 路径 bit-identical；只更新 conditional 或 unconditional 一侧的迭代 token。 - 验收：覆盖 eager 与 CUDA graph、bool 与 additive float mask、CFG/non-CFG 及不同 batch/concurrency，断言 masked 位置仍为 `-inf`、float mask 数值保留、graph/eager 输出满足既定精度合同、每步无固定输入重算和逐请求同步；单独验证 `enable_tf32` 默认关闭且显式开启前后性能/非 bit-identical 语义有记录。 ^[PR #5174] + +### EXEC-10a — NPU 平台 runner 必须保持 runtime mode 与 dummy-run 接口合同 + +- 触发：平台 runner 覆盖 `dummy_run` 或接收 `cudagraph_runtime_mode`，而共享 runner 新增运行时参数或需要校验复合 runtime mode。 +- 强制：平台 override 必须保留共享签名中的 `randomize_inputs` 默认值并将其传给 `maybe_randomize_inputs`；runtime mode 必须调用 `is_valid_runtime_mode()` 判断当前实例是否有效。 +- 禁止：通过实例调用返回集合的 `valid_runtime_modes()` 冒充实例有效性校验，或因共享签名新增可选参数而让平台 override 产生接口漂移。 +- 验收：对 base NPU runner 与 generation override 分别覆盖 `randomize_inputs` 的默认和显式路径，并用包含 `FULL_DECODE_ONLY` 等复合模式的正负用例断言 `is_valid_runtime_mode()` 的校验结果和签名 parity。 ^[PR #6096] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0dcb5ba2a..671bbbf2f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bbe6ccc5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f41451f6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 38297eb99..0d10e43f3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bbe6ccc5` 验证存在 +- 源码校验：以上路径均已在 `main @ f41451f6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index bf491f8cf..6a2313f0c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bbe6ccc5`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f41451f6`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `bbb709cf` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `bbe6ccc5` 本轮将 PR #6096 中 NPU KV connector finalize 的 PP 所有权条件，以及 `cudagraph_runtime_mode` 的有效性校验沉淀到 model-executor 规则。Docker 镜像升级属于 ci 范围未写入；`randomize_inputs` 的 dummy-run 合同已由 EXEC-1e 覆盖，未重复新增。不能据此推断所有平台 runner 都采用相同的异步输出合同。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from bbb709cf \ - --to bbe6ccc5 \ + --from bbe6ccc5 \ + --to f41451f6 \ --repo \ --mode report-only ``` From 1820bcb80b037ff414f8cf85a270e498885a2df0 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:10:00 +0800 Subject: [PATCH 140/465] knowledge(vllm-omni): sync 4b66f62e (PR #5741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 4b66f62e10f1b765dc6dc6baa5bbfe633234ec25 Upstream-PR: #5741 Knowledge-Delta: 配置解析链不再使用 `stage_configs_path`，显式 deploy YAML 统一通过 `deploy_config` 进入。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2b8ea6247..17e39fa90 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bbe6ccc512a404a2df8c977ea29003002f2683e8 + previous_audited_sha: f41451f64c8b2fc20f8296f4c2a12c7d3bb16181 audited_ref: main - audited_sha: f41451f64c8b2fc20f8296f4c2a12c7d3bb16181 + audited_sha: 4b66f62e10f1b765dc6dc6baa5bbfe633234ec25 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 614baeaf4..514e987dd 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f41451f64c8b2fc20f8296f4c2a12c7d3bb16181`. The canonical machine baseline +4b66f62e10f1b765dc6dc6baa5bbfe633234ec25`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6dfd2ea7d..fcb3fcb06 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f41451f6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4b66f62e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index b9a6b9d62..908f31635 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741"] --- # vLLM-Omni 配置开发门禁 @@ -123,6 +123,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 验收：争议字段在最终逐 stage 对象中可读回，并与第一位 consumer 一致；平台 overlay 必须同时断言受影响 stage 和未覆盖 control，不能只验证 YAML 原文。^[PR #5671] +### CONF-3b — 显式 deploy YAML 统一经 `deploy_config` 解析 + +- 触发：修改 `Omni`、`AsyncOmni`、`OmniEngineArgs`、orchestrator 参数或阶段解析链中的显式 YAML 配置入口。 +- 强制：将 `deploy_config` 作为唯一显式 deploy YAML 入口；未提供时走模型默认配置解析，并让 CLI、headless、offline、runner 和测试调用方使用同一字段，直到最终逐 stage 配置可读回。 +- 禁止：在该解析链中恢复 `stage_configs_path` 字段、双路径互斥判断、格式探测或 legacy `stage_args` 直接加载分支；不得让旧参数静默改变最终配置。 +- 验收：覆盖显式 `deploy_config`、未提供配置时的模型默认回退，以及标准/headless/offline/runner 传播路径，断言最终 stage config 一致，并确认公开参数与内部 args 不再包含 `stage_configs_path`。 ^[PR #5741] + ### CONF-4a — composable strategy 只暴露已经接通的 axis - 触发：新增 composable parallel strategy、axis、routing/LB policy 或 stage override。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2df26e085..52839285d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f41451f6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4b66f62e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8f3a0b549..8a66092bc 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f41451f6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4b66f62e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2db0abe02..3176b42c9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f41451f6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4b66f62e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 671bbbf2f..35ed95abe 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f41451f6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4b66f62e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0d10e43f3..cbb24664a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f41451f6` 验证存在 +- 源码校验：以上路径均已在 `main @ 4b66f62e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6a2313f0c..9cc304977 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f41451f6`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4b66f62e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `bbe6ccc5` 本轮将 PR #6096 中 NPU KV connector finalize 的 PP 所有权条件，以及 `cudagraph_runtime_mode` 的有效性校验沉淀到 model-executor 规则。Docker 镜像升级属于 ci 范围未写入；`randomize_inputs` 的 dummy-run 合同已由 EXEC-1e 覆盖，未重复新增。不能据此推断所有平台 runner 都采用相同的异步输出合同。 +本次相对 `f41451f6` 本轮在 `components/configuration/rules.md` 新增 `CONF-3b`，记录显式 deploy YAML 统一使用 `deploy_config` 的配置边界，并补充 PR #5741 来源。不能据此推断用户文档/模板已完成迁移，也不能推断 `resolve_model_config_path()` 或遗留 YAML loader 的后续清理已经完成。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from bbe6ccc5 \ - --to f41451f6 \ + --from f41451f6 \ + --to 4b66f62e \ --repo \ --mode report-only ``` From 18ecd2a0f451789f9e69999ac7f492816becd197 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:13:28 +0800 Subject: [PATCH 141/465] knowledge(vllm-omni): sync 83d1f114 (PR #5500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 83d1f11449a0eaf2910a0a8e34c055df20e4f169 Upstream-PR: #5500 Knowledge-Delta: 新增 LTX phase-LoRA 单 Transformer、非等长 SP attention mask，以及 audio padding 的 RNG、rescale 和 Ring 边界规则。 --- adapters/vllm_omni/release_baseline.yaml | 8 ++++---- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-attention.md | 12 ++++++++++-- .../vllm-omni/components/diffusion/rules-lora.md | 12 ++++++++++-- .../components/diffusion/rules-sp-padding.md | 12 ++++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 12 files changed, 45 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 17e39fa90..1a76546ff 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f41451f64c8b2fc20f8296f4c2a12c7d3bb16181 + previous_audited_sha: 4b66f62e10f1b765dc6dc6baa5bbfe633234ec25 audited_ref: main - audited_sha: 4b66f62e10f1b765dc6dc6baa5bbfe633234ec25 + audited_sha: 83d1f11449a0eaf2910a0a8e34c055df20e4f169 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 84 sha256: 18b2cca9644d8f2054029272c2fc5e3aa3c30e8a63ccf653d5293c1f9312c2a9 diffusion: - count: 60 - sha256: ae57a7d6867352714c00b5b93a5f22874ebc2f370cd32067925bb14ec84214f8 + count: 61 + sha256: f751159bd0647552d405a0481ad582caf8f928787b8d5c7863fe5e863e12ca08 pipelines: count: 55 sha256: 0fb5191131211a863c0f81991bfb24990682cea47d6064f8a781d25ef0eda858 diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 514e987dd..2731a148a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4b66f62e10f1b765dc6dc6baa5bbfe633234ec25`. The canonical machine baseline +83d1f11449a0eaf2910a0a8e34c055df20e4f169`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fcb3fcb06..bdb55d72d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4b66f62e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 83d1f114` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 52839285d..88365b9ea 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4b66f62e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 83d1f114` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index ff675b35f..df880bb46 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -1,10 +1,10 @@ --- title: "Diffusion attention 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py] +sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py"] confidence: high --- @@ -76,3 +76,11 @@ confidence: high rejection；真实 Ascend + updated MindIE-SD 用 irregular grid 对 dense reference。目标 NPU test 仅以 `sparsity=0` 证明 protected-tail 几何下全 block 等价，不证明稀疏选择的质量、 realized sparsity 或加速。^[PR #6000] + +## DIFF-1k — 非等长 SP attention 必须保留 K/V mask 并走 mask-safe backend + +- 触发：模型在 strict Ulysses/SP 中为 self-attention 或 cross-attention 增加 K/V padding mask，且 Q 与 K 的序列长度可能不同。 +- 强制：保留完整逻辑 K/V mask，并以 `[B, 1, 1, K]` 语义传递；masked LTX attention 必须走支持该 mask 的 SDPA fallback，unmasked 请求才保留 native backend，不能修改通用 Ulysses 的 mask 语义来适配单一模型。 +- 禁止：因为 cross-attention 的 Q/K 长度不同就丢弃 mask；把只匹配 query 长度的二维 mask 交给 Flash varlen；把 backend 的 `supports_attention_mask` 能力等同于支持任意 mask layout。 +- 验收：CPU/mock 覆盖 masked self-attention、masked unequal-length cross-attention 与 unmasked path，分别断言 SDPA/native dispatch 和 mask shape；真实 SP 再以带 padding 的 LTX 输出对照未分片参考。^[PR #5500] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md index a74f02356..e023bcaa2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -1,10 +1,10 @@ --- title: "Diffusion LoRA 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/entrypoints/test_async_omni_diffusion_config.py] +sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py"] confidence: high --- @@ -82,3 +82,11 @@ confidence: high 单 transformer、多 transformer与缺失 target；high/low 顺序必须另用固定 checkpoint 的模型级 语义/质量证据验证。目标单测覆盖双 transformer 的 load/unload，但没有真实 Wan checkpoint 数值或视频质量自动 gate。^[PR #2783] + +## DIFF-2o — LTX phase LoRA 必须绑定 recipe 并复用单一 Transformer + +- 触发：LTX 多阶段 pipeline 需要在同一 Transformer 上加载官方 raw safetensors refinement LoRA，或需要区分普通两阶段与 merged distilled 两阶段的权重路径。 +- 强制：以 recipe 的 `adapter_slot` 作为唯一 phase switch；普通两阶段使用 `None → ltx_distilled`，full-distilled 两阶段不得加载 LoRA。Sidecar 先查 model root，再按 profile 的官方文件名和仓库回退到 Hub；parser 必须逐一校验 A/B 配对、映射结果、重复目标和 shape。Adapter wrapper 要在 base 权重加载前安装并 remap 到 `base_layer`，保留 Row/Column/QKV 的 rank-local TP slice；未量化 BF16 使用 adapter dtype 中的 layer-fused `B@A`，量化 base 使用 dynamic LoRA。 +- 禁止：为 phase adapter 增加第二个 resident Transformer 或 LTX 专用环境变量；允许 request/static LoRA 与内部 phase adapter 组合；在非 BF16 或量化 base 上声称 layer-fused 可用；对缺失配对、未映射模块或不匹配 shard 静默跳过。 +- 验收：覆盖官方 key mapping、缺失 A/B、unmapped/duplicate target、dynamic 与 fused 精确算术、Row/Column/QKV 本地切片、base 权重不变、phase 进入/退出和 finalize 时机；同时验证 generic `model_paths` sidecar 覆盖与 model-root/Hub fallback。^[PR #5500] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md b/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md index 182fe653b..9be57229a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md @@ -1,10 +1,10 @@ --- title: "Diffusion SP padding 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4645", docs/user_guide/diffusion/parallelism/sequence_parallel.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/sp_plan.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/hooks/sequence_parallel.py, vllm_omni/diffusion/models/flux2/flux2_transformer.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py] +sources: ["PR #4645", docs/user_guide/diffusion/parallelism/sequence_parallel.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/sp_plan.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/hooks/sequence_parallel.py, vllm_omni/diffusion/models/flux2/flux2_transformer.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_latents.py", "vllm_omni/diffusion/models/ltx2/ltx2_denoise.py", "vllm_omni/diffusion/models/ltx2/ltx2_guidance.py"] confidence: high --- @@ -49,3 +49,11 @@ confidence: high numerical/quality parity；默认 false 仍是明确的近似策略。 - 验收：性能结论须在目标 SHA 固定硬件/backend、warmup/repeats 与 exact case 重跑；质量结论对 mask true/false 使用相同 seed/input 并设模型合适的图像/视频 metric gate。^[PR #4645] + +## SPPAD-2b — audio SP padding 必须可屏蔽、不可改变 RNG 或 sampler state + +- 触发：AV diffusion 请求的 audio latent length 不能被 SP size 整除，或 guidance/rescale 和 attention 同时消费了 SP 补齐后的 audio tensor。 +- 强制：只为逻辑 audio tokens 采样和消费 request RNG，再追加确定性的零 padding；把 valid-token mask 同时传给 audio self-attention 与 audio-to-video cross-attention，rescale 统计只使用原始 token 数，输出前清除 padding。纯 Ulysses 可使用该 mask；Ring degree 大于 1 遇到 padding 必须 fail closed。 +- 禁止：把零 padding 当作 sampler state、让 padding 消耗 request RNG、用 padding 参与 guidance rescale，或在 Ring 无法消费 key mask 时继续执行 unmasked attention。 +- 验收：覆盖 SP=1、非整除 padding、provided/generated latents、RNG 后续状态、零尾部、rescale token count、masked SDPA dispatch 和 Ring rejection；有效 audio/video 输出须与无 padding 的固定 seed 参考保持既定容差。^[PR #5500] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8a66092bc..99b326b94 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4b66f62e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 83d1f114` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3176b42c9..8844836ad 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4b66f62e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 83d1f114` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 35ed95abe..c535f02da 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4b66f62e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 83d1f114` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cbb24664a..6587c9e0a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4b66f62e` 验证存在 +- 源码校验：以上路径均已在 `main @ 83d1f114` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9cc304977..2e8e849ed 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4b66f62e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 83d1f114`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `f41451f6` 本轮在 `components/configuration/rules.md` 新增 `CONF-3b`，记录显式 deploy YAML 统一使用 `deploy_config` 的配置边界，并补充 PR #5741 来源。不能据此推断用户文档/模板已完成迁移，也不能推断 `resolve_model_config_path()` 或遗留 YAML loader 的后续清理已经完成。 +本次相对 `4b66f62e` 本轮沉淀了 LTX 两阶段 phase-LoRA、带非等长 Q/K 的 SP attention mask，以及音频 SP padding/Ring 边界规则。不应据此推断所有 diffusion 模型、量化组合或未覆盖硬件后端都具备两阶段支持或质量 parity；CI、文档、recipe 和测试改动未单独形成规则。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from f41451f6 \ - --to 4b66f62e \ + --from 4b66f62e \ + --to 83d1f114 \ --repo \ --mode report-only ``` From ca9da2d3e607f66630a9f20be74f07f64524b7ea Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:14:55 +0800 Subject: [PATCH 142/465] knowledge(vllm-omni): sync dbc0dd6d (PR #6068) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: dbc0dd6d85272aab507c78b35f837d8651807087 Upstream-PR: #6068 Knowledge-Delta: 在 configuration rules 中新增 VOMNI-CFG-1k，固化嵌套 attention 配置的投影合同与 Higgs Stage 0 原生 FlashInfer 部署门禁。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1a76546ff..b5092632b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4b66f62e10f1b765dc6dc6baa5bbfe633234ec25 + previous_audited_sha: 83d1f11449a0eaf2910a0a8e34c055df20e4f169 audited_ref: main - audited_sha: 83d1f11449a0eaf2910a0a8e34c055df20e4f169 + audited_sha: dbc0dd6d85272aab507c78b35f837d8651807087 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2731a148a..44c45630e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -83d1f11449a0eaf2910a0a8e34c055df20e4f169`. The canonical machine baseline +dbc0dd6d85272aab507c78b35f837d8651807087`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bdb55d72d..4bc06fa40 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 83d1f114` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ dbc0dd6d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 908f31635..7817cbcb9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068"] --- # vLLM-Omni 配置开发门禁 @@ -90,6 +90,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 验收：从 deploy/structured/legacy 入口读取非默认值，断言 sync 与 async thinker 的最终 stage config 和 vLLM cache consumer 均得到 `mamba_ssm_cache_dtype=float32`，并覆盖 bf16 模型、fp32 SSM state 的组合。 ^[PR #5842] +### VOMNI-CFG-1k — 模型专用 attention 子配置必须贯穿 deploy 与 typed projection + +- 触发：模型 deploy profile 需要通过 vLLM 顶层 attention backend 下的 `attention_config` 控制具体 decode kernel，或新增该嵌套字段并要求 structured、legacy、typed 路径保持一致。 +- 强制：将 `attention_config` 纳入 `_ModelEngineOverrides` 与 `OmniStageModelConfig`，确保它从 deploy profile 投影到最终 stage engine args。Higgs 的三个 profile 必须在 Stage 0 同时设置 `attention_backend: FLASHINFER` 与 `attention_config.use_trtllm_attention: false`，Stage 1 保持不变。 +- 禁止：把 `FLASHINFER` 视为原生 FlashInfer decode 的保证；依据八个 codebook 推断 attention 为 `q_len=8`；只修改一个 Higgs profile；或把 SM90/XQA capability 通过误认为 BF16 工作负载的性能结论。 +- 验收：配置 schema、structured/legacy projection 测试均能读回 `{"use_trtllm_attention": false}`；解析三个 Higgs YAML 并断言 Stage 0 的 backend 与 nested pin 一致，最终日志为 `flashinfer-native`，并用同一栈 A/B 验证 TTFP/RTF 回到原生 FlashInfer 的范围。 ^[PR #6068] + ### CONF-1a — 多 stage 共卡时 diffusion stage 必须显式设 gpu_memory_utilization - 触发：多 stage 模型 stage 共卡时，分布式测试在模型加载期 CUDA OOM。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 88365b9ea..742c479f1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 83d1f114` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ dbc0dd6d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 99b326b94..c00c0ff60 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 83d1f114` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ dbc0dd6d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8844836ad..8e73b40f4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 83d1f114` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ dbc0dd6d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c535f02da..a768e0d1d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 83d1f114` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ dbc0dd6d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6587c9e0a..a577c85e2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 83d1f114` 验证存在 +- 源码校验：以上路径均已在 `main @ dbc0dd6d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2e8e849ed..08509386b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 83d1f114`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ dbc0dd6d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `4b66f62e` 本轮沉淀了 LTX 两阶段 phase-LoRA、带非等长 Q/K 的 SP attention mask，以及音频 SP padding/Ring 边界规则。不应据此推断所有 diffusion 模型、量化组合或未覆盖硬件后端都具备两阶段支持或质量 parity；CI、文档、recipe 和测试改动未单独形成规则。 +本次相对 `83d1f114` 本轮新增配置规则，记录 `attention_config` 的 typed projection 与 Higgs Stage 0 原生 FlashInfer pin。不要据此推断所有 FlashInfer/SM90 工作负载都应禁用 XQA，也不要推断 Stage 1 或其他模型需要同样设置。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 4b66f62e \ - --to 83d1f114 \ + --from 83d1f114 \ + --to dbc0dd6d \ --repo \ --mode report-only ``` From c77862a64c706b11055081f23e6791d45facb0a0 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:18:43 +0800 Subject: [PATCH 143/465] knowledge(vllm-omni): sync 90a08c0e (PR #4765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 90a08c0edaa15eda9f0b7bc740fd226703b77470 Upstream-PR: #4765 Knowledge-Delta: 新增 dots.tts 的默认 deploy 生效、side-channel batch 对齐、多模块权重载入和连续音频侧路审查门禁。 --- adapters/vllm_omni/release_baseline.yaml | 16 ++++++++-------- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/configuration/rules.md | 9 ++++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-bridge-batch.md | 10 +++++++++- .../vllm-omni/components/model-executor/rules.md | 16 +++++++++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 12 files changed, 51 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b5092632b..8c44848d4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 83d1f11449a0eaf2910a0a8e34c055df20e4f169 + previous_audited_sha: dbc0dd6d85272aab507c78b35f837d8651807087 audited_ref: main - audited_sha: dbc0dd6d85272aab507c78b35f837d8651807087 + audited_sha: 90a08c0edaa15eda9f0b7bc740fd226703b77470 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 84 - sha256: 18b2cca9644d8f2054029272c2fc5e3aa3c30e8a63ccf653d5293c1f9312c2a9 + count: 85 + sha256: 408a354d13a1f637b91bbf945225621f2b0ccd1ef072276dccfbc6b67b8d085d diffusion: count: 61 sha256: f751159bd0647552d405a0481ad582caf8f928787b8d5c7863fe5e863e12ca08 pipelines: - count: 55 - sha256: 0fb5191131211a863c0f81991bfb24990682cea47d6064f8a781d25ef0eda858 + count: 56 + sha256: 0a3d52b792acfa8e77a8a4090b58adfbcfe52368fea9bad99bfbbff70f2983a2 deploy_yamls: - count: 83 - sha256: 015a788739d95fb9c89223067f3116bd6913e4d38a2fb96d1a25f6bdb9a47e88 + count: 84 + sha256: 4644bdac38345b7e34b3585bad19d947dd26a6ace7acb7392b22e9c86ce51e29 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 44c45630e..3632134c8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -dbc0dd6d85272aab507c78b35f837d8651807087`. The canonical machine baseline +90a08c0edaa15eda9f0b7bc740fd226703b77470`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4bc06fa40..25c46a5b7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ dbc0dd6d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 90a08c0e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 7817cbcb9..063feb77b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765"] --- # vLLM-Omni 配置开发门禁 @@ -137,6 +137,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：在该解析链中恢复 `stage_configs_path` 字段、双路径互斥判断、格式探测或 legacy `stage_args` 直接加载分支；不得让旧参数静默改变最终配置。 - 验收：覆盖显式 `deploy_config`、未提供配置时的模型默认回退，以及标准/headless/offline/runner 传播路径，断言最终 stage config 一致，并确认公开参数与内部 args 不再包含 `stage_configs_path`。 ^[PR #5741] +### CONF-3c — 模型专用 deploy profile 必须真正自动生效 + +- 触发：新增带模型专用 side-path 或多模态输出标记的 AR pipeline，并同时提供默认 deploy YAML。 +- 强制：让 model config、architecture registry、pipeline registry 与 `default_deploy_config_name` 指向同一模型；对 dots.tts 保持 `enforce_eager: true` 与 `enable_prefix_caching: false`，并从未显式传入 deploy 配置的真实 `Omni` 初始化中核对最终逐 stage 值；在 prefix-cache merge 尚未保留 `sparse_audio` 前，不得移除该 pin。 +- 禁止：以 YAML 文件存在或 pipeline key 存在推断默认配置已经自动加载；只验证原始 YAML 而跳过最终 stage config；覆盖 `enable_prefix_caching` 后仍宣称 dots.tts 输出合同不变。 +- 验收：用 HF `model_type=dots_tts` 的无显式配置启动路径断言 architecture、pipeline、默认 deploy 文件和最终 flags 一致，并完成超过一个 160 ms patch 的离线生成，确认没有 prefix-cache 导致的单 patch 截断。^[PR #4765] + ### CONF-4a — composable strategy 只暴露已经接通的 axis - 触发：新增 composable parallel strategy、axis、routing/LB policy 或 stage override。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 742c479f1..49556fc70 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ dbc0dd6d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 90a08c0e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c00c0ff60..5e0b0caa4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ dbc0dd6d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 90a08c0e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8e73b40f4..1d288ba7a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ dbc0dd6d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 90a08c0e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md index ad5afe002..eb6c7e6a9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py] +sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765"] confidence: high --- @@ -110,3 +110,11 @@ Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x ^[PR #5842] 相关执行流见 [model-executor architecture](architecture.md)；scheduler 侧的对应合同见 [scheduler 规则](../scheduler/rules.md)。 + +## EXEC-1j — side-channel 停止队列必须按 batch row 保持占位对齐 + +- 触发：有状态 AR 模型在 mixed prefill/decode batch 中通过 side-channel 队列把逐请求停止信号映射为 batch logits。 +- 强制：每个 batch row 每步都向 `_results_queue` 放入一个位置项；prefill 使用 `(req_id, None)` 占位，`compute_logits` 按原 batch position 消费并在无信号或未越过停止阈值时强制 continue，消费后清空队列。 +- 禁止：省略 prefill/new-request 占位、压缩或重排队列、把 `(continue, stop)` 概率直接当 logits，或依赖全 `-inf` 行的偶然 `argmax` 结果维持对齐。 +- 验收：用同批 prefill+decode、停止阈值以下、空队列和多请求输入测试，分别断言每行的 continue/stop 结果、请求归属与队列耗尽；batch 顺序变化不能改变停止对象。^[PR #4765] + diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 91ccce2d6..687814a02 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765"] --- # Model Executor 规则 @@ -187,6 +187,13 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 encoder 还须对任何未加载 plain retained parameter 在启动时硬失败；unknown checkpoint key 可继续告警，因为它不会留下 model parameter 未初始化。^[PR #4958] ^[PR #5777] ^[PR #5824] +### EXEC-2c — 多模块 checkpoint 载入必须按配置块与参数集闭环 + +- 触发：模型 checkpoint 将主干 LM 与 DiT、patch encoder、vocoder、speaker encoder 等辅助模块拆分到不同配置块、文件或命名空间，或需要额外的 latent statistics 文件。 +- 强制：从 checkpoint 配置块构造并校验各模块尺寸；单次遍历权重迭代器，按每个模块的精确 `state_dict` key 集合路由，并分别处理 `llm.model.*`、辅助模块和非 safetensors 统计文件；记录各模块实际加载数与期望数。 +- 禁止：缺失配置块时静默套用其他 checkpoint 的默认值；以命中一个 target name 代表整组权重完整；让 speaker 权重落入 VAE catch-all；重复消费只能遍历一次的权重迭代器，或用默认初始化掩盖未加载参数。 +- 验收：对已验证 checkpoint 逐模块核对加载计数、无 missing/extra tensor 和预期 dtype；对缺失配置块、架构常量不匹配、部分 shard 及错误命名空间执行 fail-fast 测试，并确认推理前就暴露错误。^[PR #4765] + ### EXEC-6a — shared image example 先建 canonical envelope，model-extra 只做特化变换 - 触发：修改 shared T2I/I2V example、`model_extras` prompt registry 或模型 prompt builder。 @@ -234,3 +241,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：平台 override 必须保留共享签名中的 `randomize_inputs` 默认值并将其传给 `maybe_randomize_inputs`；runtime mode 必须调用 `is_valid_runtime_mode()` 判断当前实例是否有效。 - 禁止：通过实例调用返回集合的 `valid_runtime_modes()` 冒充实例有效性校验，或因共享签名新增可选参数而让平台 override 产生接口漂移。 - 验收：对 base NPU runner 与 generation override 分别覆盖 `randomize_inputs` 的默认和显式路径，并用包含 `FULL_DECODE_ONLY` 等复合模式的正负用例断言 `is_valid_runtime_mode()` 的校验结果和签名 parity。 ^[PR #6096] + +### EXEC-11a — 连续 AR 音频侧路必须保持 fp32、上下文与正常终止 + +- 触发：vLLM-native AR 模型在每个 token/patch 旁路执行 DiT 或 flow-matching 采样、连续声码器解码，并通过 Omni multimodal output 输出增量音频。 +- 强制：Euler/ODE 累积状态保持 fp32，仅将 DiT matmul 输入置为模型 dtype；跨 patch 保留 causal decoder 的 streaming window，并在模型停止或容量边界通过正常 stop 信号前 flush lookahead；输出按 step 发 delta audio，并保留 `meta.sparse_audio` 等路由标记。 +- 禁止：以 bf16 作为多步积分默认累积 dtype；每个 patch 独立零填充解码；容量耗尽时 raise 使 engine 失效；停止时丢弃 decoder 尾部，或让 scaffold hidden 通过默认 multimodal merge 混入音频。 +- 验收：跨两次 engine restart 做 bit-identical/确定性探针，做跨 patch 首词与边界上下文验证；将容量缩小后确认请求正常截断且同一 engine 的下一次 generate 仍成功，并断言逐步 delta 合并后只有目标音频张量。^[PR #4765] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a768e0d1d..058c0fced 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ dbc0dd6d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 90a08c0e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a577c85e2..267e2f450 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ dbc0dd6d` 验证存在 +- 源码校验：以上路径均已在 `main @ 90a08c0e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 08509386b..5fc430bad 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ dbc0dd6d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 90a08c0e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `83d1f114` 本轮新增配置规则，记录 `attention_config` 的 typed projection 与 Higgs Stage 0 原生 FlashInfer pin。不要据此推断所有 FlashInfer/SM90 工作负载都应禁用 XQA，也不要推断 Stage 1 或其他模型需要同样设置。 +本次相对 `dbc0dd6d` 本轮为 configuration 与 model-executor 增加了 dots.tts 的默认 deploy 生效、混合 prefill/decode 停止信号对齐、多模块 checkpoint 载入，以及连续音频侧路的数值与输出门禁。不能据此推断 online serving、voice cloning、未验证 checkpoint 或 SamplingParams.seed 已获得支持；文档、示例和测试变更未沉淀为规则。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 83d1f114 \ - --to dbc0dd6d \ + --from dbc0dd6d \ + --to 90a08c0e \ --repo \ --mode report-only ``` From ffe83ec51915ff616306e21f3ba6402bed508ca6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:22:17 +0800 Subject: [PATCH 144/465] knowledge(vllm-omni): sync 596c16a5 (PR #5990) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 596c16a550aa134faf7f3dcfa0f8adf513ccd9ce Upstream-PR: #5990 Knowledge-Delta: 新增共享 fused Q/K RMSNorm+packed RoPE 合同，以及 MiniMax H3 与 TeaCache 对 packed RoPE table 的统一复用约束。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 10 +++++++++- 11 files changed, 30 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8c44848d4..d49f0bd88 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: dbc0dd6d85272aab507c78b35f837d8651807087 + previous_audited_sha: 90a08c0edaa15eda9f0b7bc740fd226703b77470 audited_ref: main - audited_sha: 90a08c0edaa15eda9f0b7bc740fd226703b77470 + audited_sha: 596c16a550aa134faf7f3dcfa0f8adf513ccd9ce # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3632134c8..e86eaa992 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -90a08c0edaa15eda9f0b7bc740fd226703b77470`. The canonical machine baseline +596c16a550aa134faf7f3dcfa0f8adf513ccd9ce`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 25c46a5b7..c43a4d1e5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 90a08c0e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 596c16a5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 49556fc70..eaada7979 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 90a08c0e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 596c16a5` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 7d1b29c14..0c976d534 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py"] confidence: high --- @@ -101,6 +101,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl ## Checkpoint 与分布式加载 +### DIFF-1l — fused Q/K RMSNorm 与 packed RoPE 必须共享布局并保留 eager fallback + +- 触发：共享 diffusion attention 新增或修改 Q/K RMSNorm+RoPE fused op、packed frequency table、custom-op dispatch，或 TeaCache 等 extractor 复用该边界。 +- 强制：公共 API 固定 `q/k` 为 `[tokens, heads, head_dim]`、norm weight 为 `[head_dim]`、`rope_table` 为 `[tokens, rotary_dim]`，并按 non-interleaved `[cos(theta), sin(theta)]` 存储；先完成 RMSNorm，再只旋转前 `rotary_dim`，原样保留 head tail。仅在 CUDA、Triton、BF16、`head_dim=128`、`rotary_dim=96` 时走 fused fast path，其余 device、dtype 与 geometry 必须走 eager reference；输入、权重和 table 的 dtype/device 合同必须保持一致。 +- 禁止：把模型原始或 tiled `freqs` 直接当作 packed cos/sin table，交换 cos/sin、改用 interleaved layout，或仅因 Triton/custom op 可用就强制走 fast path；fast path 也不得物化 normalized Q/K 与 rotary 中间结果来掩盖布局错误。 +- 验收：CUDA/Triton 测试以 `F.rms_norm` 加显式 packed RoPE reference 对照 q、k，并覆盖 sequence length `1`、`257`、`1024`；另须覆盖 unsupported fallback、参数校验和非 contiguous 权重/table。当前 PR 只证明 BF16 专用 geometry 的三组数值对照，未证明 fallback、H3 集成或跨硬件 parity；4×B300 上一次 warmup/一次测量的 109.687→106.205 秒仅是该 workload 的性能观察。^[PR #5990] + ### DIFF-2a — checkpoint remap 必须追到已注册且真实消费的目标 - 触发：增加或修改 weight mapper、scale 名称、quantization adapter 或 key resolution。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5e0b0caa4..ecbdf4c92 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 90a08c0e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 596c16a5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1d288ba7a..44091c7e0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 90a08c0e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 596c16a5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 058c0fced..2d62cc2b4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 90a08c0e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 596c16a5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 267e2f450..4d37f10b0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 90a08c0e` 验证存在 +- 源码校验：以上路径均已在 `main @ 596c16a5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5fc430bad..3cd74249a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 90a08c0e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 596c16a5`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `dbc0dd6d` 本轮为 configuration 与 model-executor 增加了 dots.tts 的默认 deploy 生效、混合 prefill/decode 停止信号对齐、多模块 checkpoint 载入，以及连续音频侧路的数值与输出门禁。不能据此推断 online serving、voice cloning、未验证 checkpoint 或 SamplingParams.seed 已获得支持；文档、示例和测试变更未沉淀为规则。 +本次相对 `90a08c0e` 本轮新增共享 Q/K RMSNorm+packed RoPE 的输入、布局与 fallback 审查规则，并补充 MiniMax H3 对 packed RoPE table 及 TeaCache 复用的模型约束；不得据此推断其他 dtype、设备、几何或所有 H3 端到端性能/质量均已验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from dbc0dd6d \ - --to 90a08c0e \ + --from 90a08c0e \ + --to 596c16a5 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 4c8cbfa20..738e94cd0 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py"] confidence: high --- @@ -165,3 +165,11 @@ confidence: high tests 再覆盖 malformed fallback 和 Laser math。真实 Ascend parity/内存/延迟证据不能由 CPU mock 替代；同 seed E2E output comparison 与重新抓 memory snapshot 在 PR 中仍列为 follow-up。 ^[PR #5891] ^[PR #5997] + +## MMH3-1h — H3 必须预构造 packed RoPE table 并让缓存提取器复用 + +- 触发：修改 MiniMax H3 的 `MiniMaxH3Rope`、`MiniMaxH3Attention`、SP prepare、TeaCache extractor 或 q/k fused 调用。 +- 强制：H3 先将 `MiniMaxH3Rope` 的 tiled frequency 输出交给 `_build_rope_table`，只从其前半段构造 cos/sin 并生成 BF16 packed table；该 table 必须在完整 packed sequence 上构造一次，经 `sp_prepare` 原样传给所有 DiT blocks 和 attention，TeaCache extractor 复用同一 helper。`rope_table=None` 的 token-refiner 路径继续只做 q/k RMSNorm；H3 的 128 维 head 只旋转前 96 维并保留后 32 维。共享算子边界以 [DIFF-1l](../../components/diffusion/rules.md#diff-1l) 为准。 +- 禁止：把 H3 已复制的第二个 frequency half 当成独立 sin 输入，向 fused API 传 raw `rope_freqs`，或在每个 block、SP 分支和 TeaCache extractor 内重建不同 table；也不能把 token-refiner 的无 RoPE 路径误接到 fused table 调用。 +- 验收：模型合同测试须断言完整 sequence 的 table shape 与 `[cos, sin]` 数值、SP 透传 identity、DiT q/k 的 96/128 partial rotation 和 32 维 tail preservation，并覆盖 TeaCache extractor 与 `rope_table=None` 分支。当前 PR 新增的仅是独立 fused-kernel BF16 测试，没有 H3 集成或 TeaCache 数值回归，因此不能据此宣称模型端到端 parity。^[PR #5990] + From 5814457b652578479936e4d19049d64288bcf890 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:22:48 +0800 Subject: [PATCH 145/465] knowledge(vllm-omni): sync 8a5bf2fb (PR #6187) Upstream-Commit: 8a5bf2fb3f1ab0f7242c72d25f694652fe957256 Upstream-PR: #6187 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d49f0bd88..c08acc3da 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 90a08c0edaa15eda9f0b7bc740fd226703b77470 + previous_audited_sha: 596c16a550aa134faf7f3dcfa0f8adf513ccd9ce audited_ref: main - audited_sha: 596c16a550aa134faf7f3dcfa0f8adf513ccd9ce + audited_sha: 8a5bf2fb3f1ab0f7242c72d25f694652fe957256 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e86eaa992..2011d71cb 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -596c16a550aa134faf7f3dcfa0f8adf513ccd9ce`. The canonical machine baseline +8a5bf2fb3f1ab0f7242c72d25f694652fe957256`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c43a4d1e5..fd6735608 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 596c16a5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8a5bf2fb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index eaada7979..541e8bf12 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 596c16a5` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8a5bf2fb` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ecbdf4c92..68f815cc7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 596c16a5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8a5bf2fb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 44091c7e0..b5742aaed 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 596c16a5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8a5bf2fb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2d62cc2b4..07b67ffa5 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 596c16a5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8a5bf2fb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4d37f10b0..2bc067c35 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 596c16a5` 验证存在 +- 源码校验：以上路径均已在 `main @ 8a5bf2fb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3cd74249a..f7737ce97 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 596c16a5`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8a5bf2fb`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `90a08c0e` 本轮新增共享 Q/K RMSNorm+packed RoPE 的输入、布局与 fallback 审查规则，并补充 MiniMax H3 对 packed RoPE table 及 TeaCache 复用的模型约束；不得据此推断其他 dtype、设备、几何或所有 H3 端到端性能/质量均已验证。 +本次相对 `596c16a5` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 90a08c0e \ - --to 596c16a5 \ + --from 596c16a5 \ + --to 8a5bf2fb \ --repo \ --mode report-only ``` From 9d49f99ecfe2e01174b39fd9f933648d56c282f8 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:24:38 +0800 Subject: [PATCH 146/465] knowledge(vllm-omni): sync 5d50bdc2 (PR #6165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 5d50bdc2ab7691b98fa0371ab2d9c534ca801991 Upstream-PR: #6165 Knowledge-Delta: 新增 diffusion worker spawn 前限制继承线程数的 system-runtime 规则，并保留显式 `OMP_NUM_THREADS` 覆盖能力。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c08acc3da..9aa3e44ea 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 596c16a550aa134faf7f3dcfa0f8adf513ccd9ce + previous_audited_sha: 8a5bf2fb3f1ab0f7242c72d25f694652fe957256 audited_ref: main - audited_sha: 8a5bf2fb3f1ab0f7242c72d25f694652fe957256 + audited_sha: 5d50bdc2ab7691b98fa0371ab2d9c534ca801991 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2011d71cb..786bf7119 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8a5bf2fb3f1ab0f7242c72d25f694652fe957256`. The canonical machine baseline +5d50bdc2ab7691b98fa0371ab2d9c534ca801991`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fd6735608..93f2c9b08 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8a5bf2fb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5d50bdc2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 541e8bf12..2e2121d64 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8a5bf2fb` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5d50bdc2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 0c976d534..3682694fd 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165"] confidence: high --- @@ -229,5 +229,12 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：丢弃请求步数、用 stage 总生成时间替代 DiT execution time，或把跨多个结果的 scalar 累加后再计算 per-step 值。 - 验收：覆盖完整 metadata、缺失/`None`/零步数、正步数、重复结果与零 execution time；分别断言 per-step 值精确、无效步数跳过且 scalar 不累加。^[PR #4755] +### DIFF-4m — diffusion worker spawn 前必须限制继承的线程数 + +- 触发：修改 `MultiprocDiffusionExecutor._launch_workers`、worker spawn 流程或 diffusion 多进程的 OMP/Torch 线程环境。 +- 强制：在设置 multiprocessing start method 并创建 worker 进程前调用 `set_multiprocessing_worker_envs()`；未设置 `OMP_NUM_THREADS` 时沿用其线程上限策略，显式用户值必须保留。 +- 禁止：worker 已 spawn 后才设置线程环境；覆盖用户显式的 `OMP_NUM_THREADS`；把默认线程值 1 宣称为所有 diffusion host-side work 的最优值。 +- 验收：在固定镜像中验证环境未设置时 worker 的 `OMP_NUM_THREADS` 与 `torch.get_num_threads()` 均为 1，并验证显式线程值仍生效；多 GPU 启动不得因每个 worker 继承全部可见 CPU 线程而产生 N 倍过量线程。^[PR #6165] + 相关执行流见 [Diffusion architecture](architecture.md)；benchmark 证据合同见 [performance evidence](../../benchmark/guides/performance-evidence.md)。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 68f815cc7..e53e3cb19 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8a5bf2fb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5d50bdc2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b5742aaed..e74b0404c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8a5bf2fb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5d50bdc2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 07b67ffa5..12568ca17 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8a5bf2fb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5d50bdc2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2bc067c35..e9b00bbd5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8a5bf2fb` 验证存在 +- 源码校验：以上路径均已在 `main @ 5d50bdc2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f7737ce97..3f2938a67 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8a5bf2fb`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5d50bdc2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `596c16a5` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `8a5bf2fb` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 596c16a5 \ - --to 8a5bf2fb \ + --from 8a5bf2fb \ + --to 5d50bdc2 \ --repo \ --mode report-only ``` From 76d000b3c197ce6a9fd06c935a38a305780d3ee2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:25:09 +0800 Subject: [PATCH 147/465] knowledge(vllm-omni): sync 2b08419d (PR #5593) Upstream-Commit: 2b08419d8be88f3f78852e42bb70a0bd46b289ae Upstream-PR: #5593 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9aa3e44ea..6b96582bd 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8a5bf2fb3f1ab0f7242c72d25f694652fe957256 + previous_audited_sha: 5d50bdc2ab7691b98fa0371ab2d9c534ca801991 audited_ref: main - audited_sha: 5d50bdc2ab7691b98fa0371ab2d9c534ca801991 + audited_sha: 2b08419d8be88f3f78852e42bb70a0bd46b289ae # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 786bf7119..fef7eacd8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5d50bdc2ab7691b98fa0371ab2d9c534ca801991`. The canonical machine baseline +2b08419d8be88f3f78852e42bb70a0bd46b289ae`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 93f2c9b08..9462a5c63 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5d50bdc2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2b08419d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2e2121d64..3cd6078aa 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5d50bdc2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 2b08419d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e53e3cb19..4aa2a9aa7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5d50bdc2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2b08419d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e74b0404c..f548fc462 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5d50bdc2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2b08419d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 12568ca17..aa9226cfe 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5d50bdc2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2b08419d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e9b00bbd5..8a8a32cb8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5d50bdc2` 验证存在 +- 源码校验：以上路径均已在 `main @ 2b08419d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3f2938a67..286bad3fd 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5d50bdc2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2b08419d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `8a5bf2fb` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `5d50bdc2` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 8a5bf2fb \ - --to 5d50bdc2 \ + --from 5d50bdc2 \ + --to 2b08419d \ --repo \ --mode report-only ``` From 2353fb2cb0d5b05652a934df3046961c54d0a82a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:26:45 +0800 Subject: [PATCH 148/465] knowledge(vllm-omni): sync bc73546c (PR #5886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: bc73546c065844136b16e3a8e799cc0bd36bc2ca Upstream-PR: #5886 Knowledge-Delta: 新增 EXEC-12a，记录 ROCm AR-Diffusion 分页注意力必须通过 AITER/flash-attn varlen 的 packed-KV 兼容路径执行。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6b96582bd..d0797f981 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5d50bdc2ab7691b98fa0371ab2d9c534ca801991 + previous_audited_sha: 2b08419d8be88f3f78852e42bb70a0bd46b289ae audited_ref: main - audited_sha: 2b08419d8be88f3f78852e42bb70a0bd46b289ae + audited_sha: bc73546c065844136b16e3a8e799cc0bd36bc2ca # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fef7eacd8..c3da861ac 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2b08419d8be88f3f78852e42bb70a0bd46b289ae`. The canonical machine baseline +bc73546c065844136b16e3a8e799cc0bd36bc2ca`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9462a5c63..08cab2a4d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2b08419d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bc73546c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3cd6078aa..5c32dccf5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2b08419d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bc73546c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4aa2a9aa7..5e4602ee4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2b08419d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bc73546c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f548fc462..b7e431925 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2b08419d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bc73546c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 687814a02..8ef46089a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886"] --- # Model Executor 规则 @@ -248,3 +248,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：Euler/ODE 累积状态保持 fp32，仅将 DiT matmul 输入置为模型 dtype；跨 patch 保留 causal decoder 的 streaming window，并在模型停止或容量边界通过正常 stop 信号前 flush lookahead；输出按 step 发 delta audio，并保留 `meta.sparse_audio` 等路由标记。 - 禁止：以 bf16 作为多步积分默认累积 dtype；每个 patch 独立零填充解码；容量耗尽时 raise 使 engine 失效；停止时丢弃 decoder 尾部，或让 scaffold hidden 通过默认 multimodal merge 混入音频。 - 验收：跨两次 engine restart 做 bit-identical/确定性探针，做跨 patch 首词与边界上下文验证；将容量缩小后确认请求正常截断且同一 engine 的下一次 generate 仍成功，并断言逐步 delta 合并后只有目标音频张量。^[PR #4765] + +### EXEC-12a — ROCm 分页注意力必须走 packed KV 的兼容 varlen 路径 + +- 触发：修改 `vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py` 的 ROCm 分页注意力、AITER/`flash_attn` varlen kernel、`block_table` 映射或 KV block layout。 +- 强制：ROCm 路径必须避开 CUDA 专用的 `vllm.vllm_flash_attn` 接口，优先解析 AITER、再回退到 `flash_attn`；依据每个请求的 `seq_lens` 构造 `cu_seqlens_k`，在 device 上按 `block_table` gather 可见 KV 为 packed tensors，并使用匹配的 query offsets、最大长度、`softmax_scale` 和 `causal` 参数调用 `flash_attn_varlen_func`。 +- 禁止：复用 CUDA 专用的 `seqused_k`/`fa_version` 参数合同；直接把不兼容的 ROCm paged kernel 用于 AR-Diffusion 的 frame-aligned blocks；依赖 ROCm kernel 对 `block_table` 的隐式支持；丢失逐请求长度或把 KV gather 到 CPU 后再调用 kernel。 +- 验收：在 ROCm GPU 上覆盖不同 history/action 长度、block table、请求长度和 `commit_current` 组合，与 dense reference 校验数值和输出 shape；分别验证 AITER 与 `flash_attn` fallback，确认 HIP 路径不导入 `vllm.vllm_flash_attn`，并确认 CUDA/CPU 路径行为不变。 ^[PR #5886] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index aa9226cfe..9e531e9c2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2b08419d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bc73546c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8a8a32cb8..f305aebef 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2b08419d` 验证存在 +- 源码校验：以上路径均已在 `main @ bc73546c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 286bad3fd..3aad0037d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2b08419d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bc73546c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `5d50bdc2` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `2b08419d` 本轮仅沉淀 ROCm AR-Diffusion 分页注意力的 packed-KV 与 varlen kernel 兼容合同；CI 迁移、镜像版本、测试调整和 pytest-shard 依赖均属于 CI、testing 或 tooling 范围，未写入 Model Executor 规则。该规则不表示所有 ROCm CI 队列或其他注意力实现都已验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 5d50bdc2 \ - --to 2b08419d \ + --from 2b08419d \ + --to bc73546c \ --repo \ --mode report-only ``` From 170007d7218bac14ce572666d1dbff46a8c64c8b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:28:09 +0800 Subject: [PATCH 149/465] knowledge(vllm-omni): sync b300f666 (PR #5677) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b300f666568e6b83a1e8893a3e13fa71afa88aa5 Upstream-PR: #5677 Knowledge-Delta: 新增 DIFF-2p：HSDP/FSDP2 的 FP8 权重准备必须覆盖 legacy 与新版 online linear method。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d0797f981..19e0dd2b2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2b08419d8be88f3f78852e42bb70a0bd46b289ae + previous_audited_sha: bc73546c065844136b16e3a8e799cc0bd36bc2ca audited_ref: main - audited_sha: bc73546c065844136b16e3a8e799cc0bd36bc2ca + audited_sha: b300f666568e6b83a1e8893a3e13fa71afa88aa5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c3da861ac..fdf9498ea 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bc73546c065844136b16e3a8e799cc0bd36bc2ca`. The canonical machine baseline +b300f666568e6b83a1e8893a3e13fa71afa88aa5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 08cab2a4d..6d5518f3e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bc73546c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b300f666` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5c32dccf5..9765f83fa 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bc73546c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b300f666` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 3682694fd..49b5a39d3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py"] confidence: high --- @@ -211,6 +211,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl ## 质量阈值与资源辅助 +### DIFF-2p — HSDP/FSDP2 必须同时识别 legacy 与 online FP8 linear method + +- 触发：修改 diffusion HSDP/FSDP2 参数准备、online FP8 quantization method 或非连续权重布局处理。 +- 强制：同时识别 `Fp8LinearMethod` 与 `Fp8PerTensorOnlineLinearMethod`；当 FP8 权重是转置形成的非连续 view 时，先将底层 `(out, in)` row-major storage 替换为 contiguous weight 供 FSDP2 接受，再让 kernel 的 `_get_layer_params` 返回对应转置 view，且同一 kernel 只 patch 一次。 +- 禁止：只检查 legacy `Fp8LinearMethod`、把非连续 transpose view 直接交给 `fully_shard`，或因新版 online method 已被识别就宣称所有 FP8 method 和布局均兼容。 +- 验收：legacy 与 `Fp8PerTensorOnlineLinearMethod` 都覆盖 regression test，断言重写层数、weight contiguous、`(out, in)` shape，以及 kernel 返回的转置 view shape/stride；另按 HSDP/FSDP 规则执行真实 `fully_shard` 验证。^[PR #5677] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5e4602ee4..0b4587176 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bc73546c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b300f666` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b7e431925..027a07885 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bc73546c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b300f666` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9e531e9c2..535ba4a6d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bc73546c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b300f666` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f305aebef..c1284c6b8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bc73546c` 验证存在 +- 源码校验：以上路径均已在 `main @ b300f666` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3aad0037d..b108e366a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bc73546c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b300f666`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `2b08419d` 本轮仅沉淀 ROCm AR-Diffusion 分页注意力的 packed-KV 与 varlen kernel 兼容合同；CI 迁移、镜像版本、测试调整和 pytest-shard 依赖均属于 CI、testing 或 tooling 范围，未写入 Model Executor 规则。该规则不表示所有 ROCm CI 队列或其他注意力实现都已验证。 +本次相对 `bc73546c` 本轮将 HSDP/FSDP2 对 legacy 与新版 online FP8 linear method 的兼容要求沉淀到 diffusion 共享规则。文档增补和回归测试本身不单独形成知识规则；该规则也不能外推到未列出的其他 FP8 quantization method 或已验证的通用硬件性能结论。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 2b08419d \ - --to bc73546c \ + --from bc73546c \ + --to b300f666 \ --repo \ --mode report-only ``` From cf2ac7d5f407d3a916ce325594c1fb16039c107a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:29:54 +0800 Subject: [PATCH 150/465] knowledge(vllm-omni): sync 022d7a76 (PR #6064) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 022d7a7695ffc3ce7329c38656e388cc11aecb8d Upstream-PR: #6064 Knowledge-Delta: 新增 MMH3-2i：参考视频抽样采用一次 ffmpeg 选择性 RGB 解码，decord 缺失时统一回退到 ffmpeg，帧数探测优先容器元数据并在 reshape 前校验精确字节数。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../repos/vllm-omni/models/minimax-h3/rules-media.md | 12 ++++++++++-- 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 19e0dd2b2..a96b2a8b4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bc73546c065844136b16e3a8e799cc0bd36bc2ca + previous_audited_sha: b300f666568e6b83a1e8893a3e13fa71afa88aa5 audited_ref: main - audited_sha: b300f666568e6b83a1e8893a3e13fa71afa88aa5 + audited_sha: 022d7a7695ffc3ce7329c38656e388cc11aecb8d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fdf9498ea..ddeea8da8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b300f666568e6b83a1e8893a3e13fa71afa88aa5`. The canonical machine baseline +022d7a7695ffc3ce7329c38656e388cc11aecb8d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6d5518f3e..3212d89e4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b300f666` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 022d7a76` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9765f83fa..a91168c93 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b300f666` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 022d7a76` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0b4587176..60c5040ff 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b300f666` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 022d7a76` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 027a07885..68b02a05b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b300f666` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 022d7a76` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 535ba4a6d..8608dc19c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b300f666` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 022d7a76` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c1284c6b8..a19f0c2a4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b300f666` 验证存在 +- 源码校验：以上路径均已在 `main @ 022d7a76` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b108e366a..1ff9172aa 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b300f666`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 022d7a76`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `bc73546c` 本轮将 HSDP/FSDP2 对 legacy 与新版 online FP8 linear method 的兼容要求沉淀到 diffusion 共享规则。文档增补和回归测试本身不单独形成知识规则；该规则也不能外推到未列出的其他 FP8 quantization method 或已验证的通用硬件性能结论。 +本次相对 `b300f666` 本轮在 MiniMax H3 媒体规则中新增参考视频按索引选择性解码、容器帧数元数据优先、decord 缺失时的统一 ffmpeg 回退及 RGB 字节数校验。该证据不能推断其他模型、解码器或硬件上的通用性能提升，也不能替代端到端质量验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from bc73546c \ - --to b300f666 \ + --from b300f666 \ + --to 022d7a76 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md index e654d809a..900a3a613 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md @@ -1,10 +1,10 @@ --- title: "MiniMax H3 媒体输入与精度规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5752", "PR #5829", "PR #5978", .buildkite/cuda/test-nightly.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5752", "PR #5829", "PR #5978", .buildkite/cuda/test-nightly.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064"] confidence: high --- @@ -74,3 +74,11 @@ confidence: high nightly 声明 4×H100；PR 的 I2VA 0.975190/35.981286、Ref2VA 0.980577/ 42.650204 来自 4×B300 外部运行，不是 checked-in 结果。T2VA 因严格 gate 未达标被删除，仅表示无 retained pixel-level gate，不表示功能不支持。^[PR #5978] + +## MMH3-2i — 参考视频解码必须按需求索引并闭合 RGB 字节合同 + +- 触发：修改 MiniMax H3 `reference_video.py` 的参考视频抽帧、`decord` 不可用时的回退、`ffprobe` 帧数探测或 prepared RGB stream 解码路径。 +- 强制：`_probe_video` 优先使用容器提供的 `nb_read_frames`/`nb_frames`，仅在元数据缺失时追加 `-count_frames` 扫描；Qwen 抽样必须将请求的 frame indices 交给一次 ffmpeg raw-video pipe，通过 `select` 只解码这些帧，完整回退也复用同一 ffmpeg RGB24 路径。解码前校验 frame count、宽高和 indices，必须在 reshape 前校验输出字节数等于 `frame_count × width × height × 3`。 +- 禁止：为每个采样帧启动一次 ffmpeg、无条件对高码率 prepared stream 做完整扫描、在 decord 缺失时重新使用逐帧 PyAV 全量解码，或对不完整 raw-video 输出静默 reshape；不得把该 CPU 解码优化扩展成 MUSA/NPU 专用路径或通用 decoder 性能保证。 +- 验收：contract tests 覆盖 selective/full ffmpeg command、`select` indices、decord 缺失回退、容器帧数直读与缺失时的 count-frames fallback、非法 metadata 和 incomplete output；在固定 prepared video 上将 selective 与 full RGB24 数组逐像素对照 PyAV reference，并分别核对 sampled frame 数、尺寸和 timestamps。^[PR #6064] + From c895d240d5ca36f04a85b22366dcfc76e2763286 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:31:31 +0800 Subject: [PATCH 151/465] knowledge(vllm-omni): sync 728640f4 (PR #6110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 728640f4d9bb7c1b68d646e2a4c59ce1ce45de9c Upstream-PR: #6110 Knowledge-Delta: 新增 Qwen Image 编译路径中复数 RoPE 分量应使用 torch.real/torch.imag 的跨平台 alias-guard 规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/qwen-image/rules.md | 12 ++++++++++-- 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a96b2a8b4..dd6e1e648 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b300f666568e6b83a1e8893a3e13fa71afa88aa5 + previous_audited_sha: 022d7a7695ffc3ce7329c38656e388cc11aecb8d audited_ref: main - audited_sha: 022d7a7695ffc3ce7329c38656e388cc11aecb8d + audited_sha: 728640f4d9bb7c1b68d646e2a4c59ce1ce45de9c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ddeea8da8..2fe0e7f33 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -022d7a7695ffc3ce7329c38656e388cc11aecb8d`. The canonical machine baseline +728640f4d9bb7c1b68d646e2a4c59ce1ce45de9c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3212d89e4..1590ace03 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 022d7a76` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 728640f4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a91168c93..d0623d55f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 022d7a76` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 728640f4` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 60c5040ff..816002296 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 022d7a76` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 728640f4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 68b02a05b..6b4f5d96d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 022d7a76` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 728640f4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8608dc19c..3c7e58e22 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 022d7a76` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 728640f4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a19f0c2a4..6bf85ee3f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 022d7a76` 验证存在 +- 源码校验：以上路径均已在 `main @ 728640f4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1ff9172aa..70e0cfff3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 022d7a76`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 728640f4`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b300f666` 本轮在 MiniMax H3 媒体规则中新增参考视频按索引选择性解码、容器帧数元数据优先、decord 缺失时的统一 ffmpeg 回退及 RGB 字节数校验。该证据不能推断其他模型、解码器或硬件上的通用性能提升，也不能替代端到端质量验证。 +本次相对 `022d7a76` 本轮将 Qwen Image 复数 RoPE 分量的编译兼容规则归档至模型 owner；不得据此推断所有 diffusion pipeline 都需要相同改动或已获得跨平台 parity。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b300f666 \ - --to 022d7a76 \ + --from 022d7a76 \ + --to 728640f4 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-image/rules.md b/knowledge/repos/vllm-omni/models/qwen-image/rules.md index db695706c..55b31c4aa 100644 --- a/knowledge/repos/vllm-omni/models/qwen-image/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-image/rules.md @@ -1,10 +1,10 @@ --- title: "Qwen-Image 实现规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5887", tests/e2e/accuracy/test_qwen_image.py] +sources: ["PR #5887", tests/e2e/accuracy/test_qwen_image.py, "PR #6110", "vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py"] confidence: high --- @@ -20,3 +20,11 @@ confidence: high - 验收：regional compile + dense FA 下固定完整请求至少重复运行多次，先断言同实现输出稳定，再执行 SSIM≥0.97、PSNR≥30 的跨实现 gate。PR 的 L20X run 为 0.960/28.188，未通过恢复后的两个 gate； 合并后评论仍报告 0.958，因此目标证据不能证明 flake 已消除。^[PR #5887] + +## QWENIMG-1b — 复数 RoPE 分量在编译路径使用函数式算子 + +- 触发：修改 Qwen Image `QwenImageCrossAttention` 的 complex RoPE 频率分解、regional compilation 或 MUSA 执行路径。 +- 强制：对 `vid_freqs` 和 `txt_freqs` 使用 `torch.real()`/`torch.imag()` 后再转换为 query 对应 dtype；即使问题由 MUSA 暴露，也保持 CUDA 与其他平台使用同一等价函数式表达式。 +- 禁止：在该编译路径恢复 `freqs.real`/`freqs.imag` 属性链，或仅用 `--enforce-eager` 掩盖重复 tensor alias guard；不得把该模型级修复泛化为所有 diffusion RoPE caller 已具备同等保障。 +- 验收：以 legacy 属性表达式作为 reference，验证函数式路径的值与 dtype 精确一致；在目标 MUSA 上以 Qwen Image 默认编译配置验证不再出现 `Duplicate tensors found`，并在 CUDA smoke 中确认输出 parity。^[PR #6110] + From 181f287c6e5b3723f2f02b9db8b25b6d7691832a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:46:18 +0800 Subject: [PATCH 152/465] knowledge(vllm-omni): split loader and image-envelope groups out of model-executor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit 146 halted on model-executor/rules.md crossing 32768 bytes -- the page split_risk.py has been flagging, with 90 remaining units routing to this owner. The 跨 stage bridge 与 batch 合同 section had refilled to 13562 B, and its contents show why: after EXEC-1a-1i were extracted it held EXEC-2a-2c, 5b, 6a-6b and six brand-new single-rule series, 7a through 12a. A rule opening a new series has no sibling to anchor against, so it is appended at the end of the page -- which happens to sit inside a section named for a group that no longer lives there. That is worth fixing in the driver, not only here. Two real named groups extracted, along boundaries the 审查组 table already draws: EXEC-2a-2c (loader-contract) -> rules-loader-contract.md EXEC-6a-6b (image-task-envelope) -> rules-image-task-envelope.md EXEC-5b was also misfiled: it belongs to async-output with EXEC-5a and sat at the end of the bridge section. Moved into the Async Omni section and added to its group row. Headroom after: rules.md 6572 B, new pages 28538 and 30027 B. Anchor repairs, and three distinct ways they hide: - Five qualified inbound links (docs/repository-skills.md, docs/generation-and-support-rules.md, minimax-h3/rules-loading.md, lingbot-video/rules.md, review/guides/maintainer-pattern-routing.md). - One same-directory link written as a bare rules.md#exec-2b, from model-executor/architecture.md, which the grep used for the previous five could not match because it was anchored on the model-executor/ prefix. - One link with no fragment at all: krea2/rules.md pointed the text "Model Executor EXEC-2a" at the page that rule had just left. The file exists and there is nothing to resolve, so no checker objects; the reader is simply sent to the wrong page. Two further broken anchors were found and repaired, both pre-existing rather than caused by this split: minimax-h3 -> rules-cache-task.md#mmh3-2d kept a fullwidth 、 that the slug rule strips (sync 987b60c), and minimax-h3 -> diffusion#diff-1l used a short form where the anchor is the full heading slug (sync ffe83ec). No unit is consumed and audited_sha is untouched. Gates: tree 0 errors / 12 pre-existing reminders, wiki lint 0 errors across 329 pages, release audit CLEAN, 107 intra-knowledge anchored links clean in this worktree. Co-Authored-By: Claude Opus 5 (1M context) --- .../components/model-executor/_index.md | 4 +- .../components/model-executor/architecture.md | 4 +- .../rules-image-task-envelope.md | 35 ++++++++ .../model-executor/rules-loader-contract.md | 54 +++++++++++++ .../components/model-executor/rules.md | 79 +++---------------- .../docs/generation-and-support-rules.md | 4 +- .../repos/vllm-omni/docs/repository-skills.md | 4 +- .../repos/vllm-omni/models/krea2/rules.md | 4 +- .../vllm-omni/models/lingbot-video/rules.md | 4 +- .../models/minimax-h3/rules-loading.md | 4 +- .../vllm-omni/models/minimax-h3/rules.md | 4 +- .../guides/maintainer-pattern-routing.md | 4 +- 12 files changed, 118 insertions(+), 86 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md create mode 100644 knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6b4f5d96d..f2f502042 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -36,4 +36,6 @@ sources: [] | 理解共享职责和阶段边界 | [architecture](architecture.md) | | 根据 PR 描述直达 stage config、runner preprocess、stage runtime、bridge/batch 或 loader 的规则组与第一批源码 | [rules 与代码地图](rules.md) | -| runtime info、跨 stage payload、batch 与 request RNG 合同 | [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | \ No newline at end of file +| runtime info、跨 stage payload、batch 与 request RNG 合同 | [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | +| loader 的 dtype/config 获取、fused shard 与多模块 checkpoint 载入 | [loader 合同](rules-loader-contract.md) | +| shared image example envelope 与 model_extras 参数声明 | [image task envelope 合同](rules-image-task-envelope.md) | \ No newline at end of file diff --git a/knowledge/repos/vllm-omni/components/model-executor/architecture.md b/knowledge/repos/vllm-omni/components/model-executor/architecture.md index d9bffa75d..72180c40c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/architecture.md +++ b/knowledge/repos/vllm-omni/components/model-executor/architecture.md @@ -1,7 +1,7 @@ --- title: "Model Executor 共享架构" created: 2026-07-10 -updated: 2026-09-02 +updated: 2026-09-04 type: architecture tags: [vllm-omni, components, model-executor] sources: ["PR #5610", "PR #5744", "PR #6058", docs/design/feature/omni_async_output_materialization.md, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/stage_utils.py, tests/worker/test_omni_connector_mixin.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py] @@ -78,7 +78,7 @@ KV-only sender 可只保留 KV manager，不另外创建无 consumer 的 payload 和 `gate_proj`/`up_proj`，HF 同名参数由普通 loader 直接写入，TP plan 也指向这些 分离名称。只有 310P 平台 overlay 在 `prepare_qkv_weights()` 内为本地执行临时拼接 QKV weight/bias；这不表示共享模型或 checkpoint loader 已改为 fused parameter。未来再引入 -共享 fusion 时的验收门禁见 [EXEC-2b](rules.md#exec-2b-fused-shard-必须按-source-完整性与布局数值闭环)。 +共享 fusion 时的验收门禁见 [EXEC-2b](rules-loader-contract.md#exec-2b-fused-shard-必须按-source-完整性与布局数值闭环)。 ## 怎样判断问题归属 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md new file mode 100644 index 000000000..01f18701e --- /dev/null +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md @@ -0,0 +1,35 @@ +--- +title: "image task envelope 合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, model-executor] +sources: ["PR #6049", "PR #6076", vllm_omni/model_extras/] +confidence: high +--- + +# image task envelope 合同 + +`image-task-envelope` 审查组的 `EXEC-6a`–`EXEC-6b`：shared image example 的 canonical envelope 与 `model_extras` 的模型专有参数声明。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 + +## EXEC-6a — shared image example 先建 canonical envelope，model-extra 只做特化变换 + +- 触发：修改 shared T2I/I2V example、`model_extras` prompt registry 或模型 prompt builder。 +- 强制：task runner 先构造完整 canonical dict：T2I 含 `prompt`、`modalities=["image"]`；I2V + 另含 `modalities=["video"]` 与原样 `multi_modal_data`。只有 `negative_prompt is not None` 才写 key， + 因而显式空字符串必须保留。registry 接收该 dict；无 model-specific builder 时 identity-return， + 有 builder 时只翻译模型 token/template/mm kwargs。公共 online serving handler 不参与此离线 seam。 +- 禁止：pipeline 已负责 validation/normalization 时复制 generic builder；让 registry 从零重建 task + modality；用 truthiness 丢空 negative prompt；为每个模型复制 Python example。 +- 验收：canonical builder 覆盖 omitted/empty/value negative prompt 与 PIL media identity；registry + 覆盖 Bagel、MammothModa2、Ming、VACE custom path及 unknown identity path；Cosmos3/LingBot + pipeline tests继续拥有模型专属 validation。^[PR #6049] + +## EXEC-6b — model_extras 必须声明模型专有参数与输出张量范围 + +- 触发：新增或修改 `model_extras` 中模型专有请求参数、输出张量范围，或共享视频导出对不同 pipeline 输出合同的处理时。 +- 强制：按解析后的 `model_class_name` 在 registry 中声明 `extra_body_params` 与 `output_tensor_range`，通过公开 accessor 提供；消费者按整个视频的声明合同统一转换浮点张量，未声明模型保持 `negative_one_to_one` 默认行为。 +- 禁止：把模型专有选项散落到共享 runner 的通用参数或绕过 registry；按每帧当前最小值推断范围；把已是 `[0, 1]` 的模型输出再次按 `[-1, 1]` 映射。 +- 验收：registry 测试断言目标模型的 extra-body 参数和 `zero_to_one` 范围、普通 pipeline 的默认范围；覆盖单帧与 list-valued 视频的明确范围、混合正负值的统一转换，以及最终视频导出结果。 ^[PR #6076] + +相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md new file mode 100644 index 000000000..44762f53a --- /dev/null +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md @@ -0,0 +1,54 @@ +--- +title: "loader 合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, model-executor] +sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/] +confidence: high +--- + +# loader 合同 + +`loader-contract` 审查组的 `EXEC-2a`–`EXEC-2c`：dtype 与 config 获取、fused shard 完整性、多模块 checkpoint 载入。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 + +## EXEC-2a — loader 的 dtype 与 config 获取必须显式、最小化 + +- 触发：模型 loader 构造 text encoder、VAE、transformer 或只读取 checkpoint config。 +- 强制：所有子模块显式接收目标 dtype；读取单个 config 使用精确文件/metadata 获取路径。 +- 禁止：为读取 `config.json` 同步下载整套权重；依赖默认 fp32 后再靠下游 cast 修补。 +- 验收：mock 下载层证明只请求目标 config，loader 测试断言各子模块 dtype；真实 smoke + 记录峰值显存和 dtype。Krea 2 的具体约束见 + [Krea 2 规则](../../models/krea2/rules.md)。 ^[PR #4730] + +## EXEC-2b — fused shard 必须按 source 完整性与布局数值闭环 + +> H3 text encoder 当前有 fused QKV 与 gate/up owner；Qwen3 code predictor 仍使用分离 +> projection，其 PR #4958 fusion 已由 PR #5777 回退，不能把 H3 现状外推给 Qwen。 + +- 触发：准备合并 q/k/v、gate/up 等 projection，修改 HF shard 映射、wrapper/talker + loader、packed projection TP plan 或平台 override。 +- 强制：weight 与可选 bias 都按 forward split 的同一顺序数值拼装；部分 shard 和 + 整组 shard 缺失都硬失败。bookkeeping 必须按 source shard id；expected 集合从实际 fused + owner 预置，才能捕获零 shard，不能以任一 source 命中的 target name 代表完整。所有声明 + 同一 fused module 的 consumer 必须委托给共享 loader，wrapper 并恢复 returned loaded-name + 的模型前缀；平台 override 同步使用新 + fused 属性与共享 split helper。 +- 禁止：不得用逐 tensor `default_weight_loader` 绕过 fused assembler；不得只记录 skipped + shard 后让随机初始参数继续；不得只用 shape 或 returned-name 断言顺序正确。GQA + 下错误偏移仍可形状合法。plain packed `nn.Linear` 不得声明泛化 colwise TP； + 未有 TP-aware packing/loading/split 与 TP=2 测试时，TP plan 保持空。 +- 验收：数值比较 fused 参数与 `cat([q,k,v])`/`cat([gate,up])`，覆盖 bias、 + 部分/整组缺 shard、每个 consumer 的委托和前缀；以非等 q/KV width 证明 split 能识别 + GQA 错序。平台测试或静态 guard 证明不再引用已删除的 projection 属性。H3 eager text + encoder 还须对任何未加载 plain retained parameter 在启动时硬失败；unknown checkpoint key + 可继续告警，因为它不会留下 model parameter 未初始化。^[PR #4958] ^[PR #5777] ^[PR #5824] + +## EXEC-2c — 多模块 checkpoint 载入必须按配置块与参数集闭环 + +- 触发：模型 checkpoint 将主干 LM 与 DiT、patch encoder、vocoder、speaker encoder 等辅助模块拆分到不同配置块、文件或命名空间，或需要额外的 latent statistics 文件。 +- 强制：从 checkpoint 配置块构造并校验各模块尺寸；单次遍历权重迭代器，按每个模块的精确 `state_dict` key 集合路由，并分别处理 `llm.model.*`、辅助模块和非 safetensors 统计文件；记录各模块实际加载数与期望数。 +- 禁止：缺失配置块时静默套用其他 checkpoint 的默认值；以命中一个 target name 代表整组权重完整；让 speaker 权重落入 VAE catch-all；重复消费只能遍历一次的权重迭代器，或用默认初始化掩盖未加载参数。 +- 验收：对已验证 checkpoint 逐模块核对加载计数、无 missing/extra tensor 和预期 dtype；对缺失配置块、架构常量不匹配、部分 shard 及错误命名空间执行 fail-fast 测试，并确认推理前就暴露错误。^[PR #4765] + +相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 8ef46089a..9174e0487 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -32,10 +32,10 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `core` | 每次 model-executor 审查 | `EXEC-1a` | | `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | | `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`–`EXEC-1i`，见 [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | -| `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`, `EXEC-2b` | -| `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a` | +| `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`–`EXEC-2c`，见 [loader 合同](rules-loader-contract.md) | +| `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a`, `EXEC-5b` | | `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | -| `image-task-envelope` | shared image task example 或 `model_extras` prompt builder | `EXEC-6a` | +| `image-task-envelope` | shared image task example 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6b`，见 [image task envelope 合同](rules-image-task-envelope.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | ## 严格配置校验 @@ -129,6 +129,13 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru runner 分开：CUDA/ROCm 已验证，XPU/MUSA 可能进入共享 GPU path 但未验证，Ascend NPU 的 独立 runner 仍同步 materialize。 ^[PR #5610] +### EXEC-5b — NPU KV connector finalize 必须按 PP 所有权延迟 + +- 触发：NPU AR 或 generation runner 在 speculative decoding、pipeline parallel 或 `broadcast_pp_output` 场景调用 `maybe_get_kv_connector_output`。 +- 强制：仅当 `self.speculative_config is not None` 且当前 rank 是 `get_pp_group().is_last_rank` 或 `self.broadcast_pp_output` 为真时设置 `defer_finalize=True`；AR 与 generation runner 必须保持相同条件。 +- 禁止：仅依据 speculative decoding 开启状态就在所有 PP rank 延迟 finalize，或继续使用不区分 PP 所有权的 `clear_kv_metadata` 反向条件。 +- 验收：覆盖无 speculative config、speculative config、last rank、非 last rank 和 `broadcast_pp_output` 的组合，断言两个 runner 传入 connector 的 `defer_finalize` 值及最终 finalize 时机一致。 ^[PR #6096] + ## Stage 并行度和设备容量必须一起验收 - 触发条件：修改或排查全局 CLI、per-stage override、deploy YAML、平台 overlay、stage 并行度、`runtime.devices`、设备映射或 worker 启动。 @@ -148,72 +155,6 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 ## 跨 stage bridge 与 batch 合同 -### EXEC-5b — NPU KV connector finalize 必须按 PP 所有权延迟 - -- 触发：NPU AR 或 generation runner 在 speculative decoding、pipeline parallel 或 `broadcast_pp_output` 场景调用 `maybe_get_kv_connector_output`。 -- 强制：仅当 `self.speculative_config is not None` 且当前 rank 是 `get_pp_group().is_last_rank` 或 `self.broadcast_pp_output` 为真时设置 `defer_finalize=True`；AR 与 generation runner 必须保持相同条件。 -- 禁止：仅依据 speculative decoding 开启状态就在所有 PP rank 延迟 finalize，或继续使用不区分 PP 所有权的 `clear_kv_metadata` 反向条件。 -- 验收：覆盖无 speculative config、speculative config、last rank、非 last rank 和 `broadcast_pp_output` 的组合，断言两个 runner 传入 connector 的 `defer_finalize` 值及最终 finalize 时机一致。 ^[PR #6096] - -### EXEC-2a — loader 的 dtype 与 config 获取必须显式、最小化 - -- 触发：模型 loader 构造 text encoder、VAE、transformer 或只读取 checkpoint config。 -- 强制：所有子模块显式接收目标 dtype；读取单个 config 使用精确文件/metadata 获取路径。 -- 禁止：为读取 `config.json` 同步下载整套权重；依赖默认 fp32 后再靠下游 cast 修补。 -- 验收：mock 下载层证明只请求目标 config，loader 测试断言各子模块 dtype；真实 smoke - 记录峰值显存和 dtype。Krea 2 的具体约束见 - [Krea 2 规则](../../models/krea2/rules.md)。 ^[PR #4730] - -### EXEC-2b — fused shard 必须按 source 完整性与布局数值闭环 - -> H3 text encoder 当前有 fused QKV 与 gate/up owner；Qwen3 code predictor 仍使用分离 -> projection，其 PR #4958 fusion 已由 PR #5777 回退，不能把 H3 现状外推给 Qwen。 - -- 触发：准备合并 q/k/v、gate/up 等 projection，修改 HF shard 映射、wrapper/talker - loader、packed projection TP plan 或平台 override。 -- 强制：weight 与可选 bias 都按 forward split 的同一顺序数值拼装；部分 shard 和 - 整组 shard 缺失都硬失败。bookkeeping 必须按 source shard id；expected 集合从实际 fused - owner 预置，才能捕获零 shard，不能以任一 source 命中的 target name 代表完整。所有声明 - 同一 fused module 的 consumer 必须委托给共享 loader，wrapper 并恢复 returned loaded-name - 的模型前缀；平台 override 同步使用新 - fused 属性与共享 split helper。 -- 禁止：不得用逐 tensor `default_weight_loader` 绕过 fused assembler；不得只记录 skipped - shard 后让随机初始参数继续；不得只用 shape 或 returned-name 断言顺序正确。GQA - 下错误偏移仍可形状合法。plain packed `nn.Linear` 不得声明泛化 colwise TP； - 未有 TP-aware packing/loading/split 与 TP=2 测试时，TP plan 保持空。 -- 验收：数值比较 fused 参数与 `cat([q,k,v])`/`cat([gate,up])`，覆盖 bias、 - 部分/整组缺 shard、每个 consumer 的委托和前缀；以非等 q/KV width 证明 split 能识别 - GQA 错序。平台测试或静态 guard 证明不再引用已删除的 projection 属性。H3 eager text - encoder 还须对任何未加载 plain retained parameter 在启动时硬失败；unknown checkpoint key - 可继续告警，因为它不会留下 model parameter 未初始化。^[PR #4958] ^[PR #5777] ^[PR #5824] - -### EXEC-2c — 多模块 checkpoint 载入必须按配置块与参数集闭环 - -- 触发：模型 checkpoint 将主干 LM 与 DiT、patch encoder、vocoder、speaker encoder 等辅助模块拆分到不同配置块、文件或命名空间，或需要额外的 latent statistics 文件。 -- 强制：从 checkpoint 配置块构造并校验各模块尺寸；单次遍历权重迭代器，按每个模块的精确 `state_dict` key 集合路由，并分别处理 `llm.model.*`、辅助模块和非 safetensors 统计文件；记录各模块实际加载数与期望数。 -- 禁止：缺失配置块时静默套用其他 checkpoint 的默认值；以命中一个 target name 代表整组权重完整；让 speaker 权重落入 VAE catch-all；重复消费只能遍历一次的权重迭代器，或用默认初始化掩盖未加载参数。 -- 验收：对已验证 checkpoint 逐模块核对加载计数、无 missing/extra tensor 和预期 dtype；对缺失配置块、架构常量不匹配、部分 shard 及错误命名空间执行 fail-fast 测试，并确认推理前就暴露错误。^[PR #4765] - -### EXEC-6a — shared image example 先建 canonical envelope，model-extra 只做特化变换 - -- 触发：修改 shared T2I/I2V example、`model_extras` prompt registry 或模型 prompt builder。 -- 强制：task runner 先构造完整 canonical dict：T2I 含 `prompt`、`modalities=["image"]`；I2V - 另含 `modalities=["video"]` 与原样 `multi_modal_data`。只有 `negative_prompt is not None` 才写 key， - 因而显式空字符串必须保留。registry 接收该 dict；无 model-specific builder 时 identity-return， - 有 builder 时只翻译模型 token/template/mm kwargs。公共 online serving handler 不参与此离线 seam。 -- 禁止：pipeline 已负责 validation/normalization 时复制 generic builder；让 registry 从零重建 task - modality；用 truthiness 丢空 negative prompt；为每个模型复制 Python example。 -- 验收：canonical builder 覆盖 omitted/empty/value negative prompt 与 PIL media identity；registry - 覆盖 Bagel、MammothModa2、Ming、VACE custom path及 unknown identity path；Cosmos3/LingBot - pipeline tests继续拥有模型专属 validation。^[PR #6049] - -### EXEC-6b — model_extras 必须声明模型专有参数与输出张量范围 - -- 触发：新增或修改 `model_extras` 中模型专有请求参数、输出张量范围，或共享视频导出对不同 pipeline 输出合同的处理时。 -- 强制：按解析后的 `model_class_name` 在 registry 中声明 `extra_body_params` 与 `output_tensor_range`，通过公开 accessor 提供；消费者按整个视频的声明合同统一转换浮点张量，未声明模型保持 `negative_one_to_one` 默认行为。 -- 禁止：把模型专有选项散落到共享 runner 的通用参数或绕过 registry；按每帧当前最小值推断范围；把已是 `[0, 1]` 的模型输出再次按 `[-1, 1]` 映射。 -- 验收：registry 测试断言目标模型的 extra-body 参数和 `zero_to_one` 范围、普通 pipeline 的默认范围；覆盖单帧与 list-valued 视频的明确范围、混合正负值的统一转换，以及最终视频导出结果。 ^[PR #6076] - ### EXEC-7a — Omni 输出必须是扁平的 RequestOutput 子类 - 触发：修改 `OmniRequestOutput`、stage output wrapping、`omni.generate()` 返回对象或其 serving/offline 消费者。 diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md index cdb0e5163..059f57477 100644 --- a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -1,7 +1,7 @@ --- title: "Generated docs and supported-model evidence rules" created: 2026-08-10 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, docs] sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, examples/online_serving/text_to_image/, examples/online_serving/image_to_video/, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998", "PR #6045", "PR #6049", "PR #6074", docs/features/README.md, docs/design/index.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, docs/user_guide/quantization/overview.md, docs/user_guide/quantization/bitsandbytes.md, docs/user_guide/quantization/gguf.md] @@ -96,7 +96,7 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, - 触发：修改 shared T2I/I2V README、Python client、Gradio 或 launch script。 - 强制：名称与说明按 task 而非某个默认模型表述；通用 client 的 size、steps、CFG、seed 未显式提供时 省略，让 server/model default 生效。模型专属 curl recipe 可保留，但必须标清范围。canonical task - envelope 的代码 owner 见 [EXEC-6a](../components/model-executor/rules.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。 + envelope 的代码 owner 见 [EXEC-6a](../components/model-executor/rules-image-task-envelope.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。 - 禁止：在 shared client 默认发送 Qwen-specific CFG/尺寸/steps，或因脚本默认 model 是 Qwen 就把 UI 宣称为 Qwen-only；不得把 runner/pipeline synthetic smoke 写成 released-weight quality 证据。 - 验收：README/CLI/UI/server script 名称一致；optional 字段省略与显式零/空分别测试；shell `bash -n`， diff --git a/knowledge/repos/vllm-omni/docs/repository-skills.md b/knowledge/repos/vllm-omni/docs/repository-skills.md index 77b7f2ebc..af394fed9 100644 --- a/knowledge/repos/vllm-omni/docs/repository-skills.md +++ b/knowledge/repos/vllm-omni/docs/repository-skills.md @@ -1,7 +1,7 @@ --- title: "Repository skill contributor rules" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, docs] sources: ["PR #6029", "PR #6046", .claude/skills/readme.md, .claude/skills/add-diffusion-model/SKILL.md, .claude/skills/add-tts-model/SKILL.md, .claude/skills/diffusion-perf-opt/SKILL.md, .claude/skills/precheck-pr/SKILL.md, .claude/skills/precheck-pr/references/checklists.md, .claude/skills/precheck-pr/references/examples-policy.md, .claude/skills/quantization/SKILL.md, .claude/skills/review-pr/SKILL.md, .claude/skills/vllm-omni-npu-upgrade/SKILL.md, .claude/skills/vllm-omni-test/SKILL.md, docs/contributing/README.md] @@ -60,7 +60,7 @@ confidence: high 文件名；model、checkpoint、vendor 或 family 专属的 prompt、request/output adaptation 与 launch config 必须回 production model module 或 `model_extras`，用户命令和验证证据进 task docs/recipe。 两个 skill 必须引用同一份 canonical policy；shared image runner 的生产边界继续服从 - [EXEC-6a](../components/model-executor/rules.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。 + [EXEC-6a](../components/model-executor/rules-image-task-envelope.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。 - 禁止：报告仅修改或删除的既有 model-specific example 债务；因文件名看似 generic 就接受内部只 实现一个模型合同的脚本；复制 policy 到 reviewer workflow 后分别演化；把 skill blocker 描述为 已有 CI gate。 diff --git a/knowledge/repos/vllm-omni/models/krea2/rules.md b/knowledge/repos/vllm-omni/models/krea2/rules.md index 0682dbc20..578526e06 100644 --- a/knowledge/repos/vllm-omni/models/krea2/rules.md +++ b/knowledge/repos/vllm-omni/models/krea2/rules.md @@ -1,7 +1,7 @@ --- title: "Krea 2 规则" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] sources: ["PR #4730"] @@ -48,5 +48,5 @@ confidence: high - 验收：online e2e 从公开 endpoint 返回图像；capability matrix 每格绑定当前 head 的 命令、环境和结果。 ^[PR #4730] -共享 loader 合同见 [Model Executor EXEC-2a](../../components/model-executor/rules.md)； +共享 loader 合同见 [Model Executor EXEC-2a](../../components/model-executor/rules-loader-contract.md#exec-2a-loader-的-dtype-与-config-获取必须显式最小化)； 证据分层见 [model validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md index f6f7de258..224fc2629 100644 --- a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md +++ b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md @@ -1,7 +1,7 @@ --- title: "LingBot-Video 规则" created: 2026-08-10 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/diffusion/models/lingbot_video/image_condition.py, vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/model_extras/lingbot_video.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/lingbot_video/test_request_utils.py, tests/diffusion/models/lingbot_video/test_image_condition.py, tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py, tests/entrypoints/openai_api/test_image_server.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5311", "PR #5976", "PR #6049", "Issue #5883"] @@ -63,7 +63,7 @@ sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/d 不对称也必须先统一 omitted 表示再修改。另须用同字段冲突值断言 extra_args 胜过 prompt，并用 跨表示冲突冻结 merge 后的 `width/height` > `size` > `resolution+ratio`，同时保留只有 prompt 的 offline control。Canonical envelope 的共享 ownership 与 identity/custom builder 测试见 - [EXEC-6a](../../components/model-executor/rules.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。^[PR #5976] ^[PR #6049] + [EXEC-6a](../../components/model-executor/rules-image-task-envelope.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换)。^[PR #5976] ^[PR #6049] ## LBV-2a — TI2V 的 VLM 与 VAE 必须共享一次几何对齐及同一 RNG 顺序 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md index 69d4a2dde..8dd336565 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md @@ -1,7 +1,7 @@ --- title: "MiniMax H3 加载规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py] @@ -48,4 +48,4 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection source 缺失、整组缺失和 plain parameter 缺失。真实 checkpoint 的 index/name coverage 可证明 映射覆盖，但未执行真实完整加载时，不能声称 shape/dtype/full-load 已回归；本改动本身 未改变 shape/dtype handling。共享 fused-loader 门禁见 - [EXEC-2b](../../components/model-executor/rules.md#exec-2b-fused-shard-必须按-source-完整性与布局数值闭环)。^[PR #5824] + [EXEC-2b](../../components/model-executor/rules-loader-contract.md#exec-2b-fused-shard-必须按-source-完整性与布局数值闭环)。^[PR #5824] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 738e94cd0..6debad21e 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -27,7 +27,7 @@ confidence: high | FL2VA keyframe、Ref2VA mixed reference/时域限界、shape/output matrix | [MMH3-2a](rules-media.md#mmh3-2a-taskreferenceshape-与多输出必须作为一个输入矩阵维护) | `pipeline_minimax_h3.py` → `reference_video.py` | | media limit、typed/multipart reference、HTTP 400、temp source | [MMH3-2b](rules-media.md#mmh3-2b-media-ingress-在解码前限界request-错误保持-http-400) | `api_server.py` → `serving_video.py` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | [MMH3-2c](rules-cache-task.md#mmh3-2c-conditioned-vae-的固定种子必须按实际设备隔离并恢复) | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | -| modular checkpoint、combined/partition task、两套 DiT、shared component | [MMH3-2d](rules-cache-task.md#mmh3-2d-modular-h3-的-task-selector-必须同步权重、能力与所有-dit-lifecycle) | model index discovery → startup task selection → task-specific transformer/cache lifecycle | +| modular checkpoint、combined/partition task、两套 DiT、shared component | [MMH3-2d](rules-cache-task.md#mmh3-2d-modular-h3-的-task-selector-必须同步权重能力与所有-dit-lifecycle) | model index discovery → startup task selection → task-specific transformer/cache lifecycle | | request `quality`、lossless/high、dynamic Cache-DiT | [MMH3-2e](rules-cache-task.md#mmh3-2e-quality-映射只决定-request-的-cache-dit-目标) | request sampling → quality policy → request Cache-DiT runtime → denoise | | force-refresh hint、once/repeat、reinstall key | [MMH3-2f](rules-cache-task.md#mmh3-2f-force-refresh-hint-属于-active-profile-identity) | H3 `extra_args` validation → immutable cache config → installation key/refresh context | | TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | [MMH3-2g](rules-cache-task.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state) | custom enabler → H3 extractor → module-resident hook state / per-generation reset | @@ -169,7 +169,7 @@ confidence: high ## MMH3-1h — H3 必须预构造 packed RoPE table 并让缓存提取器复用 - 触发：修改 MiniMax H3 的 `MiniMaxH3Rope`、`MiniMaxH3Attention`、SP prepare、TeaCache extractor 或 q/k fused 调用。 -- 强制：H3 先将 `MiniMaxH3Rope` 的 tiled frequency 输出交给 `_build_rope_table`，只从其前半段构造 cos/sin 并生成 BF16 packed table；该 table 必须在完整 packed sequence 上构造一次，经 `sp_prepare` 原样传给所有 DiT blocks 和 attention，TeaCache extractor 复用同一 helper。`rope_table=None` 的 token-refiner 路径继续只做 q/k RMSNorm；H3 的 128 维 head 只旋转前 96 维并保留后 32 维。共享算子边界以 [DIFF-1l](../../components/diffusion/rules.md#diff-1l) 为准。 +- 强制：H3 先将 `MiniMaxH3Rope` 的 tiled frequency 输出交给 `_build_rope_table`，只从其前半段构造 cos/sin 并生成 BF16 packed table；该 table 必须在完整 packed sequence 上构造一次，经 `sp_prepare` 原样传给所有 DiT blocks 和 attention，TeaCache extractor 复用同一 helper。`rope_table=None` 的 token-refiner 路径继续只做 q/k RMSNorm；H3 的 128 维 head 只旋转前 96 维并保留后 32 维。共享算子边界以 [DIFF-1l](../../components/diffusion/rules.md#diff-1l-fused-qk-rmsnorm-与-packed-rope-必须共享布局并保留-eager-fallback) 为准。 - 禁止：把 H3 已复制的第二个 frequency half 当成独立 sin 输入，向 fused API 传 raw `rope_freqs`，或在每个 block、SP 分支和 TeaCache extractor 内重建不同 table；也不能把 token-refiner 的无 RoPE 路径误接到 fused table 调用。 - 验收：模型合同测试须断言完整 sequence 的 table shape 与 `[cos, sin]` 数值、SP 透传 identity、DiT q/k 的 96/128 partial rotation 和 32 维 tail preservation，并覆盖 TeaCache extractor 与 `rope_table=None` 分支。当前 PR 新增的仅是独立 fused-kernel BF16 测试，没有 H3 集成或 TeaCache 数值回归，因此不能据此宣称模型端到端 parity。^[PR #5990] diff --git a/knowledge/repos/vllm-omni/review/guides/maintainer-pattern-routing.md b/knowledge/repos/vllm-omni/review/guides/maintainer-pattern-routing.md index c7f98ae77..522e823a1 100644 --- a/knowledge/repos/vllm-omni/review/guides/maintainer-pattern-routing.md +++ b/knowledge/repos/vllm-omni/review/guides/maintainer-pattern-routing.md @@ -1,7 +1,7 @@ --- title: "PR intent 到 maintainer owner 路由" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-04 type: guide tags: [vllm-omni, review] sources: ["InferMatrixCopilot Issue #24", "PR #3576", "PR #3642", "PR #4106", "PR #4281", "PR #4341", "PR #4718", "PR #4730", "PR #4980", "PR #5001", "PR #5031", "PR #5037", "PR #5052", "PR #5084", "PR #5087", "PR #5088", "PR #5136", "PR #5157", "PR #6049"] @@ -37,7 +37,7 @@ diff 的 producer、consumer、失败路径与链接规则一致。 | PR 描述或风险信号 | 主 owner | 重点 | |---|---|---| | 任意模型名、别名、architecture、pipeline/model registry key | [Model resolver](../../models/catalog.md#direct-模型代码入口) | 搜三处 registry → folder/module/class → AR/diffusion models 目录 | -| `model_extras`、shared image-task examples、canonical prompt envelope | [Model Executor rules](../../components/model-executor/rules.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换) | example owns task envelope；registry 只做 model-specific transform；再按命中模型追加 owner | +| `model_extras`、shared image-task examples、canonical prompt envelope | [Model Executor rules](../../components/model-executor/rules-image-task-envelope.md#exec-6a-shared-image-example-先建-canonical-envelopemodel-extra-只做特化变换) | example owns task envelope；registry 只做 model-specific transform；再按命中模型追加 owner | | HunyuanImage3、`extra_body`、AR prompt/tokenizer | [HunyuanImage3 rules](../../models/hunyuan-image3/rules.md) | registry、server/offline seam、默认值与 fallback | | `tests/diffusion/quantization`、可选包 import、硬件支持文档 | [CI environment](../../ci/guides/ci-environment-gotchas.md) | 未安装环境、真实 kernel、claim 一致性 | | benchmark 脚本、percentile、warmup、replica isolation | [performance evidence](../../benchmark/guides/performance-evidence.md) | 计时、统计、失败退出、完整 key set | From fbf5d76a4146bc75f87e3bf1931222906ac2d805 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 04:58:08 +0800 Subject: [PATCH 153/465] knowledge(vllm-omni): sync ffd11c18 (PR #6061) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: ffd11c181e5fa4e558ba2570f326fb41371c0421 Upstream-PR: #6061 Knowledge-Delta: 新增 NPU 平台模型补丁生产注册验收，以及 MiniMax H3 Qwen3-VL 文本编码器 BNSD/BSND RoPE 和逐样本 M-RoPE 合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 9 ++++++++- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index dd6e1e648..1b8f3cf73 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 022d7a7695ffc3ce7329c38656e388cc11aecb8d + previous_audited_sha: 728640f4d9bb7c1b68d646e2a4c59ce1ce45de9c audited_ref: main - audited_sha: 728640f4d9bb7c1b68d646e2a4c59ce1ce45de9c + audited_sha: ffd11c181e5fa4e558ba2570f326fb41371c0421 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2fe0e7f33..5a6b40c09 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -728640f4d9bb7c1b68d646e2a4c59ce1ce45de9c`. The canonical machine baseline +ffd11c181e5fa4e558ba2570f326fb41371c0421`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1590ace03..844aa9cfa 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 728640f4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ffd11c18` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d0623d55f..6751f0679 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 728640f4` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ffd11c18` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 816002296..de3ebf928 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 728640f4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ffd11c18` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f2f502042..aa9fb7c05 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 728640f4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ffd11c18` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 9174e0487..655249702 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py"] --- # Model Executor 规则 @@ -196,3 +196,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：ROCm 路径必须避开 CUDA 专用的 `vllm.vllm_flash_attn` 接口，优先解析 AITER、再回退到 `flash_attn`；依据每个请求的 `seq_lens` 构造 `cu_seqlens_k`，在 device 上按 `block_table` gather 可见 KV 为 packed tensors，并使用匹配的 query offsets、最大长度、`softmax_scale` 和 `causal` 参数调用 `flash_attn_varlen_func`。 - 禁止：复用 CUDA 专用的 `seqused_k`/`fa_version` 参数合同；直接把不兼容的 ROCm paged kernel 用于 AR-Diffusion 的 frame-aligned blocks；依赖 ROCm kernel 对 `block_table` 的隐式支持；丢失逐请求长度或把 KV gather 到 CPU 后再调用 kernel。 - 验收：在 ROCm GPU 上覆盖不同 history/action 长度、block table、请求长度和 `commit_current` 组合，与 dense reference 校验数值和输出 shape；分别验证 AITER 与 `flash_attn` fallback，确认 HIP 路径不导入 `vllm.vllm_flash_attn`，并确认 CUDA/CPU 路径行为不变。 ^[PR #5886] + +### EXEC-13a — NPU 模型补丁必须由生产初始化路径注册并可回归 + +- 触发：在 NPU platform 初始化中加入模型专有 monkey patch、融合算子或其他必须全局启用的 model hook。 +- 强制：把 hook 封装成幂等的 `apply_*_patch()`，在 `adapt_patch(is_global_patch=True)` 完成后由生产初始化路径显式调用；注册必须只指向目标模型 consumer，并保留非 NPU和其他模型路径。测试必须验证生产注册链，而不只直接调用 helper。 +- 禁止：依赖 import side effect 或手工调用 helper 代替平台注册；只覆盖 patched function 的数值而不验证注册实际生效；把模型专有 patch 扩大到 MiniMax H3 DiT 或所有 NPU 模型。 +- 验收：从 NPU 平台初始化/注册入口运行测试，断言目标 Qwen3-VL text encoder 的 consumer 已替换且重复初始化不重复 patch；随后用真实 consumer 覆盖至少一个 BNSD fast path、一个 BSND fallback 和 batched M-RoPE。^[PR #6061] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3c7e58e22..a4ec44f67 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 728640f4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ffd11c18` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6bf85ee3f..a46b29208 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 728640f4` 验证存在 +- 源码校验：以上路径均已在 `main @ ffd11c18` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 70e0cfff3..a3321de7c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 728640f4`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ffd11c18`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `022d7a76` 本轮将 Qwen Image 复数 RoPE 分量的编译兼容规则归档至模型 owner；不得据此推断所有 diffusion pipeline 都需要相同改动或已获得跨平台 parity。 +本次相对 `728640f4` 本轮沉淀 PR #6061 的两条门禁：NPU 模型补丁必须通过生产初始化路径注册，以及 MiniMax H3 Qwen3-VL 文本编码器必须保持 BNSD/BSND RoPE 与逐样本 M-RoPE 语义。测试文件仅作为验收证据；不能据此推断所有 NPU 模型、其他平台或 H3 DiT RoPE 已获得相同覆盖或性能保证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 022d7a76 \ - --to 728640f4 \ + --from 728640f4 \ + --to ffd11c18 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 6debad21e..908beac02 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py"] confidence: high --- @@ -173,3 +173,10 @@ confidence: high - 禁止：把 H3 已复制的第二个 frequency half 当成独立 sin 输入，向 fused API 传 raw `rope_freqs`，或在每个 block、SP 分支和 TeaCache extractor 内重建不同 table；也不能把 token-refiner 的无 RoPE 路径误接到 fused table 调用。 - 验收：模型合同测试须断言完整 sequence 的 table shape 与 `[cos, sin]` 数值、SP 透传 identity、DiT q/k 的 96/128 partial rotation 和 32 维 tail preservation，并覆盖 TeaCache extractor 与 `rope_table=None` 分支。当前 PR 新增的仅是独立 fused-kernel BF16 测试，没有 H3 集成或 TeaCache 数值回归，因此不能据此宣称模型端到端 parity。^[PR #5990] +## MMH3-1i — Qwen3-VL 文本编码器 NPU RoPE 必须保持 BNSD 与逐样本 M-RoPE + +- 触发：修改 MiniMax H3 Qwen3-VL text encoder 在 Ascend NPU 上的 `_apply_rotary_pos_emb`、`npu_rotary_mul_with_bsnd_fallback`、BNSD/BSND layout 或逐样本 M-RoPE。 +- 强制：q/k 保持 BNSD `[batch, heads, seq, head_dim]`，`cos/sin` 保持逐样本 `[batch, seq, dim]` 并通过 `unsqueeze_dim=1` 对齐；优先使用 CANN 支持的 BNSD fused path，仅在不支持的 BNSD tiling shape 下回退 BSND，同时保留每个样本的 M-RoPE 频率，不改 MiniMax H3 DiT 的 `RotaryEmbedding` 路径。 +- 禁止：把 batch-specific frequency 广播、展平或转置成单样本语义；绕过共享 helper 直接调用 NPU 算子而失去 fallback；用 helper 直测代替生产 platform hook 验证，或把该 text-encoder 优化外推到其他 H3 RoPE 路径和平台。 +- 验收：NPU correctness tests 覆盖 BNSD fast shapes、BSND fallback shapes 和不同样本的真实 M-RoPE positions，分别以 NeoX reference 核对 q/k 输出并断言样本频率确实不同；另从平台注册入口验证生产 consumer 已启用 fused path。^[PR #6061] + From c997e63efc4fe9cb883b7fac9d75c8f4e56eaa06 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:00:16 +0800 Subject: [PATCH 154/465] knowledge(vllm-omni): sync 8ecd1f6d (PR #6167) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 8ecd1f6d5cc91aab8a475a861213720b336e2f65 Upstream-PR: #6167 Knowledge-Delta: 新增 MMH3-4a，记录 H3 Qwen3-VL 在 NPU 上以 packed gate/up projection 配合融合 SwiGLU 的实现与生产注册验收要求。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 8 +++++++- 10 files changed, 20 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1b8f3cf73..6a4e806cb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 728640f4d9bb7c1b68d646e2a4c59ce1ce45de9c + previous_audited_sha: ffd11c181e5fa4e558ba2570f326fb41371c0421 audited_ref: main - audited_sha: ffd11c181e5fa4e558ba2570f326fb41371c0421 + audited_sha: 8ecd1f6d5cc91aab8a475a861213720b336e2f65 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5a6b40c09..59aad27a5 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ffd11c181e5fa4e558ba2570f326fb41371c0421`. The canonical machine baseline +8ecd1f6d5cc91aab8a475a861213720b336e2f65`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 844aa9cfa..5e736c786 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ffd11c18` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8ecd1f6d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6751f0679..c808e4f0b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ffd11c18` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8ecd1f6d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index de3ebf928..23cf8eb19 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ffd11c18` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8ecd1f6d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index aa9fb7c05..f5cabb371 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ffd11c18` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8ecd1f6d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a4ec44f67..dc4747d89 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ffd11c18` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8ecd1f6d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a46b29208..b41cad212 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ffd11c18` 验证存在 +- 源码校验：以上路径均已在 `main @ 8ecd1f6d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a3321de7c..1b8570312 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ffd11c18`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8ecd1f6d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `728640f4` 本轮沉淀 PR #6061 的两条门禁：NPU 模型补丁必须通过生产初始化路径注册，以及 MiniMax H3 Qwen3-VL 文本编码器必须保持 BNSD/BSND RoPE 与逐样本 M-RoPE 语义。测试文件仅作为验收证据；不能据此推断所有 NPU 模型、其他平台或 H3 DiT RoPE 已获得相同覆盖或性能保证。 +本次相对 `ffd11c18` 本轮新增 MiniMax H3 的 NPU Qwen3-VL 文本 MLP 融合 SwiGLU 规则，并登记 PR #6167 来源。该提交的数值测试不能推断生产 platform hook 已注册成功，也不能外推到其他模型、平台或端到端性能。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 728640f4 \ - --to ffd11c18 \ + --from ffd11c18 \ + --to 8ecd1f6d \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 908beac02..74f8170d0 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167"] confidence: high --- @@ -180,3 +180,9 @@ confidence: high - 禁止：把 batch-specific frequency 广播、展平或转置成单样本语义；绕过共享 helper 直接调用 NPU 算子而失去 fallback；用 helper 直测代替生产 platform hook 验证，或把该 text-encoder 优化外推到其他 H3 RoPE 路径和平台。 - 验收：NPU correctness tests 覆盖 BNSD fast shapes、BSND fallback shapes 和不同样本的真实 M-RoPE positions，分别以 NeoX reference 核对 q/k 输出并断言样本频率确实不同；另从平台注册入口验证生产 consumer 已启用 fused path。^[PR #6061] +## MMH3-4a — H3 Qwen3-VL NPU 文本 MLP 必须使用 packed gate/up 与融合 SwiGLU + +- 触发：修改 MiniMax H3 Qwen3-VL text encoder 的 MLP、`gate_up_proj` packing、NPU fused operator 或对应 platform override。 +- 强制：NPU 路径使用一次 `F.linear(x, self.gate_up_proj.weight)` 生成 packed gate/up，再调用 `torch_npu.npu_swiglu(gate_up, dim=-1)`，最后复用既有 `down_proj`；gate/up 顺序和结果须等价于 `F.silu(gate) * up`，且补丁只作用于 NPU 目标 text MLP，CPU/CUDA 保持原路径。 +- 禁止：在 NPU 上切分 packed 权重后发起两次 `F.linear` 或使用未融合的 SiLU 乘法；只直接调用 helper 而不验证生产注册链；把该优化扩展到 H3 DiT 或其他模型。 +- 验收：NPU 测试以多个 packed shape 对照 split projection reference，并从真实 diffusion runner 初始化入口断言目标 `MiniMaxH3Qwen3VLTextMLP.forward` 已替换且重复初始化幂等；未通过生产入口的 helper 数值测试不能作为 hook 生效证据。^[PR #6167] From 5883455e02e3ffb3a066b1ab75d7986367a36cd1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:01:20 +0800 Subject: [PATCH 155/465] knowledge(vllm-omni): sync f7a51460 (PR #3805) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: f7a5146093bc4d16b5607901a8a2509d785777db Upstream-PR: #3805 Knowledge-Delta: 新增 SERV-4m：API server 直接入口必须复用 FlexibleArgumentParser，并将 positional model_tag 同步到实际 model 参数。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 10 +++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 22 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6a4e806cb..9828676eb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ffd11c181e5fa4e558ba2570f326fb41371c0421 + previous_audited_sha: 8ecd1f6d5cc91aab8a475a861213720b336e2f65 audited_ref: main - audited_sha: 8ecd1f6d5cc91aab8a475a861213720b336e2f65 + audited_sha: f7a5146093bc4d16b5607901a8a2509d785777db # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 59aad27a5..263fd8f7c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8ecd1f6d5cc91aab8a475a861213720b336e2f65`. The canonical machine baseline +f7a5146093bc4d16b5607901a8a2509d785777db`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5e736c786..fe461ea64 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8ecd1f6d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f7a51460` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c808e4f0b..77de5678b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8ecd1f6d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f7a51460` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 23cf8eb19..838bfcfea 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8ecd1f6d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f7a51460` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f5cabb371..16fb4f4b8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8ecd1f6d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f7a51460` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index dc4747d89..0831c588c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8ecd1f6d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f7a51460` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b41cad212..e0b91df9b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8ecd1f6d` 验证存在 +- 源码校验：以上路径均已在 `main @ f7a51460` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 6112d20ee..448ac080d 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805"] confidence: high --- @@ -269,3 +269,11 @@ confidence: high `sampling_params_list[i].lora_request` 拥有，默认参数对象不被污染。GPU 验收必须先确认 adapter ID 已加载，再证明 deterministic token/logprob 与 base 不同；#5369 另报的 `AsyncOmni.add_lora` control-RPC 反序列化成 list 问题不在本修复范围。^[PR #5374] + +### SERV-4m — API server 必须规范化 CLI 参数并同步 positional model + +- 触发：修改 `vllm_omni.entrypoints.openai.api_server` 的 CLI parser、模型参数来源或直接运行入口。 +- 强制：直接运行 API server 时复用 upstream `FlexibleArgumentParser`，使下划线参数与连字符参数保持一致的规范化行为；解析后若 `args.model_tag` 非空，必须将 `args.model` 同步为该值，以确保 positional model 不会退回 vLLM `ModelConfig` 的默认模型。 +- 禁止：使用不支持 upstream flag normalization 的独立 argparse parser；仅在 `args.model` 缺失时反向填充 `model_tag`，导致 positional model 仍未传入实际 model consumer；把 `--omni` 的兼容占位参数解释为改变直接入口运行模式的开关。 +- 验收：分别验证 positional model、`--model`、`--gpu_memory_utilization` 等下划线参数和连字符参数均可解析，并断言 `args.model` 与 `args.model_tag` 一致且服务加载请求模型，不会回退到默认 LLM。^[PR #3805] + diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1b8570312..b85ab1567 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8ecd1f6d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f7a51460`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `ffd11c18` 本轮新增 MiniMax H3 的 NPU Qwen3-VL 文本 MLP 融合 SwiGLU 规则，并登记 PR #6167 来源。该提交的数值测试不能推断生产 platform hook 已注册成功，也不能外推到其他模型、平台或端到端性能。 +本次相对 `8ecd1f6d` 本轮将 API server 的 CLI 参数规范化与 positional model/model_tag 同步约束补充到 serving 请求合同规则中。该规则不意味着模型 owner 需要变更，也不覆盖 Omni 模式默认值或模型内部加载逻辑。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from ffd11c18 \ - --to 8ecd1f6d \ + --from 8ecd1f6d \ + --to f7a51460 \ --repo \ --mode report-only ``` From 1f6215249b1867baf79f9fbb9bc63c879bdec5fe Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:01:53 +0800 Subject: [PATCH 156/465] knowledge(vllm-omni): sync 4b6d6d5c (PR #6175) Upstream-Commit: 4b6d6d5cf79506d2a6ea697da11f1b2bdceccda8 Upstream-PR: #6175 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9828676eb..d702f3425 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8ecd1f6d5cc91aab8a475a861213720b336e2f65 + previous_audited_sha: f7a5146093bc4d16b5607901a8a2509d785777db audited_ref: main - audited_sha: f7a5146093bc4d16b5607901a8a2509d785777db + audited_sha: 4b6d6d5cf79506d2a6ea697da11f1b2bdceccda8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 263fd8f7c..27e634795 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f7a5146093bc4d16b5607901a8a2509d785777db`. The canonical machine baseline +4b6d6d5cf79506d2a6ea697da11f1b2bdceccda8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fe461ea64..d83541371 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f7a51460` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4b6d6d5c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 77de5678b..655142152 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f7a51460` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4b6d6d5c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 838bfcfea..2bd9bc7ce 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f7a51460` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4b6d6d5c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 16fb4f4b8..ad14be42c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f7a51460` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4b6d6d5c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0831c588c..50ae16372 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f7a51460` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4b6d6d5c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e0b91df9b..fda0c2732 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f7a51460` 验证存在 +- 源码校验：以上路径均已在 `main @ 4b6d6d5c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b85ab1567..402a6e05b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f7a51460`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4b6d6d5c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `8ecd1f6d` 本轮将 API server 的 CLI 参数规范化与 positional model/model_tag 同步约束补充到 serving 请求合同规则中。该规则不意味着模型 owner 需要变更，也不覆盖 Omni 模式默认值或模型内部加载逻辑。 +本次相对 `f7a51460` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 8ecd1f6d \ - --to f7a51460 \ + --from f7a51460 \ + --to 4b6d6d5c \ --repo \ --mode report-only ``` From 2755dcd6936602e38132f36abaf79be3bc261291 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:03:36 +0800 Subject: [PATCH 157/465] knowledge(vllm-omni): sync fe54ecc3 (PR #5878) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: fe54ecc38845bd2a14419c57bf7c0a7a761e7161 Upstream-PR: #5878 Knowledge-Delta: 新增 SERV-1g，固化 TTS voice/VoiceID 归一化、speaker 能力列表和 default 占位行为。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d702f3425..4b317a99e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f7a5146093bc4d16b5607901a8a2509d785777db + previous_audited_sha: 4b6d6d5cf79506d2a6ea697da11f1b2bdceccda8 audited_ref: main - audited_sha: 4b6d6d5cf79506d2a6ea697da11f1b2bdceccda8 + audited_sha: fe54ecc38845bd2a14419c57bf7c0a7a761e7161 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 27e634795..44fe46af7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4b6d6d5cf79506d2a6ea697da11f1b2bdceccda8`. The canonical machine baseline +fe54ecc38845bd2a14419c57bf7c0a7a761e7161`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d83541371..3403ffed5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4b6d6d5c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fe54ecc3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 655142152..dc195c024 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4b6d6d5c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ fe54ecc3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2bd9bc7ce..afc7afead 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4b6d6d5c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fe54ecc3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ad14be42c..7a4d6d633 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4b6d6d5c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fe54ecc3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 50ae16372..51964c158 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4b6d6d5c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fe54ecc3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fda0c2732..8bea0fa26 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4b6d6d5c` 验证存在 +- 源码校验：以上路径均已在 `main @ fe54ecc3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 448ac080d..d13ce39dd 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878"] confidence: high --- @@ -119,6 +119,13 @@ confidence: high - 验收：覆盖 non-stream、HTTP streaming、WebSocket、无/有 aligner、短/超限 alignment 和多句音频；分别断言 400、header/frame 内容、超限标记，以及音频在时间戳缺失或超限时仍按合同返回。 ^[PR #4795] +## SERV-1g — TTS voice 输入、占位 default 与可用 speaker 必须闭环 + +- 触发：修改 OpenAI speech 的 `voice` 输入校验、VoiceID 归一化、可用 voice 列表、上传 speaker 或无 speaker 模型的默认 voice 语义。 +- 强制：协议层同时接受字符串和包含 `id` 的 VoiceID 对象，并在 serving 校验前归一化为小写名称；可用 voice 列表必须统一包含内置 speaker、上传 speaker 和 `default` 占位项。无实际 `default` speaker 时，`default` 请求必须被接受并转换为后端忽略的无 voice 请求；存在同名真实 speaker 时必须优先使用真实 speaker；其他非法名称返回结构化 400。 +- 禁止：要求没有可用 speaker 的模型提供一个实际存在的 voice 名称；让 voice 列表遗漏 `default` 或上传 speaker；把 `default` 占位值传入后端作为真实 speaker；仅支持字符串而拒绝合法 VoiceID 对象。 +- 验收：覆盖无上传 speaker 时的字符串和 `{\"id\": ...}` 请求、未知 voice 的 400、上传前后 `default` 与新增 speaker 的列表和语音请求，以及注册并删除同名 `default` 后占位行为恢复；另需验证同名上传 voice 的实际使用，而不只断言请求成功。 ^[PR #5878] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 402a6e05b..425e39dd9 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4b6d6d5c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fe54ecc3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `f7a51460` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `4b6d6d5c` 本轮新增 Serving 侧 TTS voice 输入、可用 speaker 列表与 `default` 占位语义规则，来源为 PR #5878。不可据此推断模型内部说话人实现或测试覆盖已证明上传的 `default` 一定被实际使用。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from f7a51460 \ - --to 4b6d6d5c \ + --from 4b6d6d5c \ + --to fe54ecc3 \ --repo \ --mode report-only ``` From c4100f61b8ac54f00f6e0ea161a58f86b7c36f75 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:04:09 +0800 Subject: [PATCH 158/465] knowledge(vllm-omni): sync b95c7374 (PR #6172) Upstream-Commit: b95c73742d3a8a2485b464bb3925cdcf1e18588a Upstream-PR: #6172 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4b317a99e..15d09bfac 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4b6d6d5cf79506d2a6ea697da11f1b2bdceccda8 + previous_audited_sha: fe54ecc38845bd2a14419c57bf7c0a7a761e7161 audited_ref: main - audited_sha: fe54ecc38845bd2a14419c57bf7c0a7a761e7161 + audited_sha: b95c73742d3a8a2485b464bb3925cdcf1e18588a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 44fe46af7..be8f5a920 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -fe54ecc38845bd2a14419c57bf7c0a7a761e7161`. The canonical machine baseline +b95c73742d3a8a2485b464bb3925cdcf1e18588a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3403ffed5..eb90f32df 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fe54ecc3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b95c7374` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index dc195c024..4728fcd56 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fe54ecc3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b95c7374` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index afc7afead..e45d24163 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fe54ecc3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b95c7374` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7a4d6d633..b6bb65b81 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fe54ecc3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b95c7374` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 51964c158..24c895e18 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fe54ecc3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b95c7374` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8bea0fa26..5233b2625 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fe54ecc3` 验证存在 +- 源码校验：以上路径均已在 `main @ b95c7374` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 425e39dd9..4a63ab5e7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fe54ecc3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b95c7374`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `4b6d6d5c` 本轮新增 Serving 侧 TTS voice 输入、可用 speaker 列表与 `default` 占位语义规则，来源为 PR #5878。不可据此推断模型内部说话人实现或测试覆盖已证明上传的 `default` 一定被实际使用。 +本次相对 `fe54ecc3` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 4b6d6d5c \ - --to fe54ecc3 \ + --from fe54ecc3 \ + --to b95c7374 \ --repo \ --mode report-only ``` From 7558fd11e5aecf86fab0ed33cb0e74d1ba4a8230 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:04:44 +0800 Subject: [PATCH 159/465] knowledge(vllm-omni): sync d671e6b0 (PR #5994) Upstream-Commit: d671e6b0db1fe93d59efa23b435d51f0215f779e Upstream-PR: #5994 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 15d09bfac..14dc34c3d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fe54ecc38845bd2a14419c57bf7c0a7a761e7161 + previous_audited_sha: b95c73742d3a8a2485b464bb3925cdcf1e18588a audited_ref: main - audited_sha: b95c73742d3a8a2485b464bb3925cdcf1e18588a + audited_sha: d671e6b0db1fe93d59efa23b435d51f0215f779e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index be8f5a920..2c98f3134 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b95c73742d3a8a2485b464bb3925cdcf1e18588a`. The canonical machine baseline +d671e6b0db1fe93d59efa23b435d51f0215f779e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index eb90f32df..4b1dbdc92 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b95c7374` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d671e6b0` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 4728fcd56..8ee8948c0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b95c7374` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d671e6b0` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e45d24163..9fb0728b8 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b95c7374` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d671e6b0` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b6bb65b81..1eec91075 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b95c7374` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d671e6b0` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 24c895e18..a6c22c89d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b95c7374` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d671e6b0` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5233b2625..d85b09bf3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b95c7374` 验证存在 +- 源码校验：以上路径均已在 `main @ d671e6b0` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4a63ab5e7..a07dae7bf 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b95c7374`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d671e6b0`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `fe54ecc3` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `b95c7374` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from fe54ecc3 \ - --to b95c7374 \ + --from b95c7374 \ + --to d671e6b0 \ --repo \ --mode report-only ``` From 4c7d092f7fa0eb6f436de8dd213aeee0b8e79e23 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:08:27 +0800 Subject: [PATCH 160/465] knowledge(vllm-omni): sync 35cacf02 (PR #5666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 35cacf020c823da56e5ad437128bc4e93a5a3ed1 Upstream-PR: #5666 Knowledge-Delta: 为 Gepard 接入补充 pipeline/架构闭环、AR runner 预算与 seed 元数据，以及 codec 音频增量输出的审查门禁。 --- adapters/vllm_omni/release_baseline.yaml | 16 ++++++++-------- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/configuration/rules.md | 9 ++++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-bridge-batch.md | 9 ++++++++- .../vllm-omni/components/model-executor/rules.md | 9 ++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 12 files changed, 43 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 14dc34c3d..03f8d7edf 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b95c73742d3a8a2485b464bb3925cdcf1e18588a + previous_audited_sha: d671e6b0db1fe93d59efa23b435d51f0215f779e audited_ref: main - audited_sha: d671e6b0db1fe93d59efa23b435d51f0215f779e + audited_sha: 35cacf020c823da56e5ad437128bc4e93a5a3ed1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 85 - sha256: 408a354d13a1f637b91bbf945225621f2b0ccd1ef072276dccfbc6b67b8d085d + count: 86 + sha256: 039140fb0a8e95f13015d11af550db8ed16e6071448889f243c1525de5dff493 diffusion: count: 61 sha256: f751159bd0647552d405a0481ad582caf8f928787b8d5c7863fe5e863e12ca08 pipelines: - count: 56 - sha256: 0a3d52b792acfa8e77a8a4090b58adfbcfe52368fea9bad99bfbbff70f2983a2 + count: 57 + sha256: 1cfc25b2c2bb9a543a77830477a795ec59e44d4abd3d07c9b8f6f05ec07d1aac deploy_yamls: - count: 84 - sha256: 4644bdac38345b7e34b3585bad19d947dd26a6ace7acb7392b22e9c86ce51e29 + count: 85 + sha256: 7cf1393ec0975196de259e9b5840c1f5465b8e9d76517c5bcdd2dbb8ce738afc # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2c98f3134..a3e4f22e5 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d671e6b0db1fe93d59efa23b435d51f0215f779e`. The canonical machine baseline +35cacf020c823da56e5ad437128bc4e93a5a3ed1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4b1dbdc92..341e603e7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d671e6b0` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 35cacf02` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 063feb77b..dac6e3893 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666"] --- # vLLM-Omni 配置开发门禁 @@ -144,6 +144,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：以 YAML 文件存在或 pipeline key 存在推断默认配置已经自动加载；只验证原始 YAML 而跳过最终 stage config；覆盖 `enable_prefix_caching` 后仍宣称 dots.tts 输出合同不变。 - 验收：用 HF `model_type=dots_tts` 的无显式配置启动路径断言 architecture、pipeline、默认 deploy 文件和最终 flags 一致，并完成超过一个 160 ms patch 的离线生成，确认没有 prefix-cache 导致的单 patch 截断。^[PR #4765] +### CONF-3d — checkpoint 身份与模型架构路由必须分层闭合 + +- 触发：checkpoint 的 `config.json` 自报为上游 backbone，但 Omni 需要独立的 model architecture、pipeline 和默认 deploy 配置时。 +- 强制：让 model registry、pipeline registry、`model_arch`、`default_deploy_config_name` 与 deploy stage 的 `hf_overrides.architectures` 闭合指向同一模型；保留 backbone 的 `model_type`，并让无显式配置的入口解析打包的 deploy YAML。 +- 禁止：只注册 pipeline 或放置 YAML 就依赖 `model_type` 推断；用 `hf_overrides.model_type` 替代 stage architecture 路由；要求离线入口额外传参来弥补默认配置未接通。 +- 验收：真实无显式 deploy 配置的初始化能读回最终 pipeline 与 architecture，backbone `get_text_config()` 仍解析为原模型类型，并覆盖 registry、默认 YAML 与最终 stage config 的一致性。 ^[PR #5666] + ### CONF-4a — composable strategy 只暴露已经接通的 axis - 触发：新增 composable parallel strategy、axis、routing/LB policy 或 stage override。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8ee8948c0..88bdbef47 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d671e6b0` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 35cacf02` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9fb0728b8..0d93a979f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d671e6b0` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 35cacf02` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1eec91075..ecef3a5e2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d671e6b0` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 35cacf02` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md index eb6c7e6a9..ca5ca8360 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765"] +sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666"] confidence: high --- @@ -118,3 +118,10 @@ Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x - 禁止：省略 prefill/new-request 占位、压缩或重排队列、把 `(continue, stop)` 概率直接当 logits，或依赖全 `-inf` 行的偶然 `argmax` 结果维持对齐。 - 验收：用同批 prefill+decode、停止阈值以下、空队列和多请求输入测试，分别断言每行的 continue/stop 结果、请求归属与队列耗尽；batch 顺序变化不能改变停止对象。^[PR #4765] +## EXEC-1k — AR runner 必须传递逐请求预算与内部采样 seed + +- 触发：共享 AR runner 把逐请求 metadata 交给 `preprocess`，而模型在 `forward` 内采样或必须在 `max_tokens`/上下文容量的最后一个有效 step 发送 multimodal payload 时。 +- 强制：从每个请求的 `sampling_params` 读取 `max_tokens` 与 `seed`，随 prompt 长度、已计算 token 数和 prefill 状态传入所有 preprocess 路径；模型用 scheduler cursor 与 `max_model_len` 判定最后输出，并将 seed 交给 request-local generator。 +- 禁止：用当前 span 或模型本地计数器猜预算；只让 vLLM 外层 sampler 看到 seed；依赖 `on_requests_finished` 之后再 flush，此时 request 已离开输出 batch，payload 无法路由。 +- 验收：通过真实 runner 的 mixed-batch 路径覆盖停止结束、预算结束和最后一个 in-flight request，断言每个请求的 committed frame 全部送达；覆盖同 seed、异 seed 及有无 batch 邻居的结果与 request 归属。 ^[PR #5666] + diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 655249702..86bfeebf5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666"] --- # Model Executor 规则 @@ -190,6 +190,13 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 禁止：以 bf16 作为多步积分默认累积 dtype；每个 patch 独立零填充解码；容量耗尽时 raise 使 engine 失效；停止时丢弃 decoder 尾部，或让 scaffold hidden 通过默认 multimodal merge 混入音频。 - 验收：跨两次 engine restart 做 bit-identical/确定性探针，做跨 patch 首词与边界上下文验证；将容量缩小后确认请求正常截断且同一 engine 的下一次 generate 仍成功，并断言逐步 delta 合并后只有目标音频张量。^[PR #4765] +### EXEC-11b — codec 音频流必须由逐请求帧账本驱动 + +- 触发：native-AR 音频模型逐步产生多头 codec codes，按 chunk 解码并通过 sparse multimodal output 交付增量 waveform 时。 +- 强制：按 request id 保存 frame history 与 `emitted_frames` 账本；仅提交有效音频 frame；每个 decode window 带 lookback 并在裁剪后用实际 decode 结果推导 samples-per-frame；以原 request id 排队增量 payload，在 STOP 或预算最后一步、request 仍在输出 batch 时完成最终 flush，并保留 `meta.sparse_audio`。 +- 禁止：在裸 chunk 边界解码、假定标称帧率或固定 hop、每帧解码或每次发送完整历史；把尾部音频放进 finished cleanup；让已离开输出 batch 的 request 继续产生不可路由 payload，或以 pooler/hidden state 充当音频。 +- 验收：CPU codec stand-in 覆盖跨 first/steady chunk 边界的完整帧拼接、lookback 和 partial final chunk；真实 runner 覆盖 mixed requests、sparse routing、预算尾部与 cleanup；GPU smoke 断言 waveform 样本数等于 committed frames 的实际 frame ledger。 ^[PR #5666] + ### EXEC-12a — ROCm 分页注意力必须走 packed KV 的兼容 varlen 路径 - 触发：修改 `vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py` 的 ROCm 分页注意力、AITER/`flash_attn` varlen kernel、`block_table` 映射或 KV block layout。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a6c22c89d..d3c7701fe 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d671e6b0` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 35cacf02` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d85b09bf3..ad9c4f169 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d671e6b0` 验证存在 +- 源码校验：以上路径均已在 `main @ 35cacf02` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a07dae7bf..34ff5d31a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d671e6b0`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 35cacf02`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b95c7374` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `d671e6b0` 以下 inventory 发生变化：autoregressive、deploy_yamls、pipelines；不得据此推断其他合同不变。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b95c7374 \ - --to d671e6b0 \ + --from d671e6b0 \ + --to 35cacf02 \ --repo \ --mode report-only ``` From a11b91828195625591b3cd1063f337c25f5748ea Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:10:39 +0800 Subject: [PATCH 161/465] knowledge(vllm-omni): sync 6f36fd3b (PR #5036) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 6f36fd3b7838c4de3a4b9d4de90c3bba7776c91e Upstream-PR: #5036 Knowledge-Delta: 新增对象存储模型 URI 的入口绕过预下载、配置物化与 basename 路由校验规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/configuration/rules.md | 9 ++++++++- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-upstream-compat.md | 12 ++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 31 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 03f8d7edf..680855560 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d671e6b0db1fe93d59efa23b435d51f0215f779e + previous_audited_sha: 35cacf020c823da56e5ad437128bc4e93a5a3ed1 audited_ref: main - audited_sha: 35cacf020c823da56e5ad437128bc4e93a5a3ed1 + audited_sha: 6f36fd3b7838c4de3a4b9d4de90c3bba7776c91e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a3e4f22e5..9612047e5 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -35cacf020c823da56e5ad437128bc4e93a5a3ed1`. The canonical machine baseline +6f36fd3b7838c4de3a4b9d4de90c3bba7776c91e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 341e603e7..d451df3ea 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 35cacf02` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6f36fd3b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index dac6e3893..efc577fbc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036"] --- # vLLM-Omni 配置开发门禁 @@ -242,3 +242,10 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 强制：采用 MOSS-TTS-Local 官方示例参数：`temperature=1.7`、`top_p=0.8`、`top_k=25`、`repetition_penalty=1.0`；如需变更，必须有模型级行为证据支持。 - 禁止：沿用通用默认采样值，或仅凭经验调整敏感采样参数而不核对官方示例。 - 验收：解析 `vllm_omni/deploy/moss_tts_local.yaml`，断言上述参数与官方示例一致，并确认最终 stage 配置实际读取这些值。 ^[PR #6156] + +### CONF-7a — 对象存储模型解析必须物化配置而保留原始 URI + +- 触发：父进程在 stage-specific `ModelConfig` 初始化前，需要从 `s3://`、`gs://` 或上游 `is_runai_obj_uri` 支持的对象存储 URI 推断模型类型、读取 HF 配置或解析 `model_index.json`。 +- 强制：通过 `is_runai_obj_uri` 将对象存储模型的 `*.model`、`*.py`、`*.json` 轻量文件按 URI 缓存一次到确定性的 `model_streamer/` 目录；`get_config`、`config.json` 和 `model_index.json` 的读取统一使用该本地目录，同时让各 stage 继续接收原始 URI 以保留 `model_weights` 的流式加载语义。 +- 禁止：把原始对象存储 URI 交给 Hugging Face 仓库解析；把临时 `model_streamer/` 路径传播成 stage 的模型身份；用完整 URI、bucket 或组织名参与模型类型 basename 匹配。 +- 验收：用 mock 覆盖对象存储 URI 的单次物化、配置驱动的 pipeline 选择、`model_index.json` 回退和 basename-only 匹配，并确认欺骗性 bucket 名不能劫持 pipeline；真实对象存储运行另行验证。^[PR #5036] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 88bdbef47..2046f7135 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 35cacf02` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6f36fd3b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0d93a979f..8aa68a443 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 35cacf02` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6f36fd3b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ecef3a5e2..07e1452d7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 35cacf02` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6f36fd3b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d3c7701fe..6813264eb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 35cacf02` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6f36fd3b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ad9c4f169..ebaead45a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 35cacf02` 验证存在 +- 源码校验：以上路径均已在 `main @ 6f36fd3b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md index c251cec2a..4586f6cc5 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md @@ -1,10 +1,10 @@ --- title: "Serving upstream 兼容规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py] +sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, "PR #5036"] confidence: high --- @@ -33,3 +33,11 @@ confidence: high 长期 public API。 - 验收：新/旧 renderer、两条 Mistral import 和有/无 stale counter 的 request layout 都能初始化， 且新版既有 counter 原值不变。^[PR #5957] + +## SERV-7c — 对象存储 URI 必须在 serving 入口绕过预下载 + +- 触发：修改 `omni_snapshot_download()` 或入口侧模型解析，使其处理 Run:AI 对象存储 URI、HF repo id 和本地模型路径。 +- 强制：使用上游 `is_runai_obj_uri` 作为唯一对象存储判断，在 Hugging Face/ModelScope 预下载前原样返回对象存储 URI；本地路径与普通 HF repo id 保持既有分支，并将后续配置读取交给配置层的本地物化逻辑。 +- 禁止：维护与 vLLM streamer 分离的本地 scheme allowlist；将对象存储 URI 当作 HF repo id 下载；让 bucket 或组织路径段参与 basename 模型匹配。 +- 验收：参数化 mock 测试覆盖 `s3://`、`gs://` 及上游支持的 `az://`，断言 HF 下载未调用且返回值未改变；同时覆盖本地路径、HF repo id 和欺骗性 bucket/basename 匹配，确认入口语义没有回归。^[PR #5036] + diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 34ff5d31a..89a303b43 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 35cacf02`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6f36fd3b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `d671e6b0` 以下 inventory 发生变化：autoregressive、deploy_yamls、pipelines；不得据此推断其他合同不变。 +本次相对 `35cacf02` 本轮记录了 Run:AI 对象存储模型 URI 在入口与配置解析阶段的保留、轻量文件物化和 basename 路由约束。现有测试主要是 mock 验证，不能据此推断真实对象存储端到端成功，也不能推断 Qwen3-TTS 的 speech_tokenizer 辅助权重缺口已解决。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from d671e6b0 \ - --to 35cacf02 \ + --from 35cacf02 \ + --to 6f36fd3b \ --repo \ --mode report-only ``` From 32fa4e3928ece029a338616391101edb5ef201af Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:11:13 +0800 Subject: [PATCH 162/465] knowledge(vllm-omni): sync 1260ceaa (PR #6075) Upstream-Commit: 1260ceaae639437372797c710b61b4584d36b9ca Upstream-PR: #6075 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 680855560..529a47bf2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 35cacf020c823da56e5ad437128bc4e93a5a3ed1 + previous_audited_sha: 6f36fd3b7838c4de3a4b9d4de90c3bba7776c91e audited_ref: main - audited_sha: 6f36fd3b7838c4de3a4b9d4de90c3bba7776c91e + audited_sha: 1260ceaae639437372797c710b61b4584d36b9ca # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9612047e5..85509d67e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6f36fd3b7838c4de3a4b9d4de90c3bba7776c91e`. The canonical machine baseline +1260ceaae639437372797c710b61b4584d36b9ca`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d451df3ea..99e075967 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6f36fd3b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1260ceaa` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2046f7135..d4fc31bf4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6f36fd3b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1260ceaa` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8aa68a443..82f167a7c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6f36fd3b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1260ceaa` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 07e1452d7..ccb33e552 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6f36fd3b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1260ceaa` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6813264eb..d50528449 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6f36fd3b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1260ceaa` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ebaead45a..483a745bd 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6f36fd3b` 验证存在 +- 源码校验：以上路径均已在 `main @ 1260ceaa` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 89a303b43..d118047e4 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6f36fd3b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1260ceaa`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `35cacf02` 本轮记录了 Run:AI 对象存储模型 URI 在入口与配置解析阶段的保留、轻量文件物化和 basename 路由约束。现有测试主要是 mock 验证，不能据此推断真实对象存储端到端成功，也不能推断 Qwen3-TTS 的 speech_tokenizer 辅助权重缺口已解决。 +本次相对 `6f36fd3b` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 35cacf02 \ - --to 6f36fd3b \ + --from 6f36fd3b \ + --to 1260ceaa \ --repo \ --mode report-only ``` From 0d9ff119365c4c6c8f3c1383537865fd51324d2d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:13:02 +0800 Subject: [PATCH 163/465] knowledge(vllm-omni): sync a70d4d4b (PR #4845) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: a70d4d4b3c0989714cbe1385a2cb2142b0233c6d Upstream-PR: #4845 Knowledge-Delta: 新增 DIFF-4n：diffusion worker 消费 overflow marker 后必须无限等待完整 multipart，禁止有限超时重试导致消息错配或丢失。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/rules.md | 10 +++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 22 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 529a47bf2..e314344f9 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6f36fd3b7838c4de3a4b9d4de90c3bba7776c91e + previous_audited_sha: 1260ceaae639437372797c710b61b4584d36b9ca audited_ref: main - audited_sha: 1260ceaae639437372797c710b61b4584d36b9ca + audited_sha: a70d4d4b3c0989714cbe1385a2cb2142b0233c6d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 85509d67e..d1a4d17df 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1260ceaae639437372797c710b61b4584d36b9ca`. The canonical machine baseline +a70d4d4b3c0989714cbe1385a2cb2142b0233c6d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 99e075967..2905b93a6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1260ceaa` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a70d4d4b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d4fc31bf4..092287bbb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1260ceaa` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ a70d4d4b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 49b5a39d3..d222c63c1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845"] confidence: high --- @@ -245,3 +245,11 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl 相关执行流见 [Diffusion architecture](architecture.md)；benchmark 证据合同见 [performance evidence](../../benchmark/guides/performance-evidence.md)。 + +### DIFF-4n — 大体积广播溢出消息必须无损配对接收 + +- 触发：修改 `WorkerProc` 忙循环、`MessageQueue` 广播接收或 SHM overflow marker 与 socket multipart 的配对逻辑。 +- 强制：消费 overflow marker 后必须通过 `recv_message()` 调用 `self.mq.dequeue(indefinite=True)`，等待对应的完整 socket payload 到达后才能继续接收下一条消息，并保持 FIFO 顺序。 +- 禁止：对已消费 marker 的 multipart 使用有限超时；超时后重新 `dequeue` 并把后续 marker 或 inline 消息与迟到 payload 配对；仅把有限超时调大作为修复。 +- 验收：单测断言接收调用使用 `indefinite=True`，并让 overflow payload 延迟超过任何有限超时后再发送 inline 消息，断言 worker 按 `overflow-1`、`inline-2` 顺序处理。当前 PR 的 0.1 秒延迟仍在旧 1 秒超时内，且 monkeypatch 的模块变量未被生产代码读取，不能单独证明旧实现会失败。^[PR #4845] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 82f167a7c..75d052688 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1260ceaa` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a70d4d4b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ccb33e552..f0d5b3bb8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1260ceaa` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a70d4d4b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d50528449..9798e880a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1260ceaa` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a70d4d4b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 483a745bd..01335d048 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1260ceaa` 验证存在 +- 源码校验：以上路径均已在 `main @ a70d4d4b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d118047e4..0790f5716 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1260ceaa`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a70d4d4b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `6f36fd3b` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `1260ceaa` 本轮沉淀了 diffusion worker 对大体积 SHM 溢出广播消息的无损、按序接收合同；不应据此推断所有 MessageQueue 超时场景或已完成通用大载荷端到端验证。测试文件对应的 testing/CI 候选仍未写入知识库。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 6f36fd3b \ - --to 1260ceaa \ + --from 1260ceaa \ + --to a70d4d4b \ --repo \ --mode report-only ``` From aec105c3f33286959945d1e7d1ac8fc4d718c999 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:15:37 +0800 Subject: [PATCH 164/465] knowledge(vllm-omni): sync 7b76b644 (PR #6173) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 7b76b6446d29adc23f563b75f604159ab11ecac8 Upstream-PR: #6173 Knowledge-Delta: 新增 strict-Ulysses 局部 sequence gate，以及 MiniMax-H3 局部 embedding、RoPE 和 projection-before-gather 的审查合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 9 ++++++++- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e314344f9..50f5da28d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1260ceaae639437372797c710b61b4584d36b9ca + previous_audited_sha: a70d4d4b3c0989714cbe1385a2cb2142b0233c6d audited_ref: main - audited_sha: a70d4d4b3c0989714cbe1385a2cb2142b0233c6d + audited_sha: 7b76b6446d29adc23f563b75f604159ab11ecac8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d1a4d17df..ce416d0f6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -a70d4d4b3c0989714cbe1385a2cb2142b0233c6d`. The canonical machine baseline +7b76b6446d29adc23f563b75f604159ab11ecac8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2905b93a6..17411293c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a70d4d4b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7b76b644` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 092287bbb..f1f9074b4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a70d4d4b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 7b76b644` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index d222c63c1..3da044f9f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173"] confidence: high --- @@ -108,6 +108,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：把模型原始或 tiled `freqs` 直接当作 packed cos/sin table，交换 cos/sin、改用 interleaved layout，或仅因 Triton/custom op 可用就强制走 fast path；fast path 也不得物化 normalized Q/K 与 rotary 中间结果来掩盖布局错误。 - 验收：CUDA/Triton 测试以 `F.rms_norm` 加显式 packed RoPE reference 对照 q、k，并覆盖 sequence length `1`、`257`、`1024`；另须覆盖 unsupported fallback、参数校验和非 contiguous 权重/table。当前 PR 只证明 BF16 专用 geometry 的三组数值对照，未证明 fallback、H3 集成或跨硬件 parity；4×B300 上一次 warmup/一次测量的 109.687→106.205 秒仅是该 workload 的性能观察。^[PR #5990] +### DIFF-1m — 局部 SP 边界必须由已注册 strict-Ulysses hook 显式证明 + +- 触发：修改 diffusion 模型的 rank-local sequence boundary、`local_sp_prepare`、SP layout 判定或 TeaCache extractor 的 `_embed()` 调用。 +- 强制：局部路径必须同时确认已注册 `sp_input---local_sp_prepare` hook、forward context 可用、`get_ulysses_mode() == "strict"`、sequence-parallel world size 大于 1 且等于 Ulysses size、Ring/AllGather size 均为 1，并确认 `seq_len` 可均匀切分；任一条件不满足都使用完整 sequence。需要完整上下文的 TeaCache extractor 必须显式传 `local_span=(0, seq_len)`。 +- 禁止：仅凭 SP rank/world size 推断局部 embedding 安全；让 Ring、AllGather-KV、advanced UAA、缺少 hook、非 forward context 或不可整除长度误走局部路径；把模型专属的局部 row 重建合同泛化为所有 diffusion 模型。 +- 验收：覆盖 strict hook 的 rank span、缺 hook、forward context 不可用、SP=1、advanced UAA、Ring、AllGather-KV 和非整除长度 fallback，并回归 TeaCache extractor 的完整 span；当前合同只由 MiniMax-H3 caller 证明，其他模型仍需独立验证。 ^[PR #6173] + ### DIFF-2a — checkpoint remap 必须追到已注册且真实消费的目标 - 触发：增加或修改 weight mapper、scale 名称、quantization adapter 或 key resolution。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 75d052688..afe687a7c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a70d4d4b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7b76b644` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f0d5b3bb8..d1e14de5d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a70d4d4b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7b76b644` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9798e880a..d4cb58587 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a70d4d4b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 7b76b644` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 01335d048..d926c3cc0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a70d4d4b` 验证存在 +- 源码校验：以上路径均已在 `main @ 7b76b644` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0790f5716..480a67b2a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a70d4d4b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7b76b644`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `1260ceaa` 本轮沉淀了 diffusion worker 对大体积 SHM 溢出广播消息的无损、按序接收合同；不应据此推断所有 MessageQueue 超时场景或已完成通用大载荷端到端验证。测试文件对应的 testing/CI 候选仍未写入知识库。 +本次相对 `a70d4d4b` 本轮新增 Diffusion 执行边界规则与 MiniMax H3 严格 Ulysses 局部 embedding/compact gather 规则，并登记 PR #6173 与 TeaCache extractor 来源。不得据此推断 21 倍端到端加速、其他模型或其他并行拓扑的支持；性能和质量结论仅绑定 PR 报告的精确 workload。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 1260ceaa \ - --to a70d4d4b \ + --from a70d4d4b \ + --to 7b76b644 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 74f8170d0..ae343bde7 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173"] confidence: high --- @@ -180,6 +180,13 @@ confidence: high - 禁止：把 batch-specific frequency 广播、展平或转置成单样本语义；绕过共享 helper 直接调用 NPU 算子而失去 fallback；用 helper 直测代替生产 platform hook 验证，或把该 text-encoder 优化外推到其他 H3 RoPE 路径和平台。 - 验收：NPU correctness tests 覆盖 BNSD fast shapes、BSND fallback shapes 和不同样本的真实 M-RoPE positions，分别以 NeoX reference 核对 q/k 输出并断言样本频率确实不同；另从平台注册入口验证生产 consumer 已启用 fused path。^[PR #6061] +## MMH3-1j — strict Ulysses 必须保持局部 embedding 与 projection-before-gather 的边界 + +- 触发：修改 MiniMax-H3 的 strict Ulysses、`_embed()`、packed RoPE、`local_sp_prepare`、DiT final projection 或 `sp_gather` 边界。 +- 强制：只有共享 strict-Ulysses gate 通过时，按 rank 的 packed span 先筛选 image/audio latent rows；token refiner 仍处理完整 text context 后再筛选 text rows；构造局部 embedding 与对应局部 RoPE rows，使用 `local_sp_prepare` 运行 blocks。局部 transformer 输出必须先执行 final projection，再拼接 video/audio logits 并 gather compact FP32 rows；恢复后按 architecture output width 拆分并执行原有 target/condition row selection。其余路径保留完整 embedding、RoPE、`sp_prepare`、hidden gather 后 projection 的旧顺序。 +- 禁止：在局部路径 gather `[S, 5376]` BF16 hidden state；绕过 `sp_input---local_sp_prepare` hook；只截取 text embedding 而不先运行完整 refiner；修改 Ulysses Q/K/V all-to-all、attention backend、packed-prefix 或 scheduler 语义；把约 21 倍 gather payload 降低写成端到端性能保证。 +- 验收：模型合同测试须证明各 rank 的局部 multimodal rows 拼接等于完整 embedding、RoPE 行数与 span 对齐、所有 fallback 恢复旧路径，并断言 projection-before-gather 的 payload width 与最终 logits/target/condition rows 等价；性能和 video/audio quality 只能在固定 PR workload、硬件、steps、seed 与 gate 下复核，不能外推到其他 task 或拓扑。 ^[PR #6173] + ## MMH3-4a — H3 Qwen3-VL NPU 文本 MLP 必须使用 packed gate/up 与融合 SwiGLU - 触发：修改 MiniMax H3 Qwen3-VL text encoder 的 MLP、`gate_up_proj` packing、NPU fused operator 或对应 platform override。 From 813bd6e5323e5b92ffbe21eae6fadc62731c9a5d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:16:43 +0800 Subject: [PATCH 165/465] knowledge(vllm-omni): sync e57574ae (PR #6152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: e57574ae290b42a1a341c86f0d1cecdb563e3187 Upstream-PR: #6152 Knowledge-Delta: 补充 OmniRequestOutput 与真实 RequestOutput 的字段 parity 及 stage output 非空字段传播校验。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 50f5da28d..071386191 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a70d4d4b3c0989714cbe1385a2cb2142b0233c6d + previous_audited_sha: 7b76b6446d29adc23f563b75f604159ab11ecac8 audited_ref: main - audited_sha: 7b76b6446d29adc23f563b75f604159ab11ecac8 + audited_sha: e57574ae290b42a1a341c86f0d1cecdb563e3187 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ce416d0f6..34361d689 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -7b76b6446d29adc23f563b75f604159ab11ecac8`. The canonical machine baseline +e57574ae290b42a1a341c86f0d1cecdb563e3187`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 17411293c..7952fea8f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7b76b644` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e57574ae` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f1f9074b4..4cf80e24d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7b76b644` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e57574ae` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index afe687a7c..6273c5ede 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7b76b644` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e57574ae` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index d1e14de5d..f88c23560 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7b76b644` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e57574ae` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 86bfeebf5..4e5f3f0ab 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152"] --- # Model Executor 规则 @@ -162,6 +162,13 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 禁止：重新引入 `request_output` 嵌套字段、动态 pass-through property、修改 dataclass 生成的 `__init__` 或递归 `unwrap` 兼容层；不得在 serving、example 或工具代码中恢复 `output.request_output` 访问。 - 验收：用真实 `RequestOutput` 和 `OmniRequestOutput` 分别验证 factory 复制的 prompt、token、outputs、finished 及 images/trajectory 内容；覆盖文本、音频、图像和 pipeline stage 的直接字段消费，并确认未完成 stage 的 `finished` 不被包装过程改写。^[PR #5146] +### EXEC-7b — OmniRequestOutput 必须保持 RequestOutput 字段与复制合同 + +- 触发：修改 `OmniRequestOutput`、升级 vLLM `RequestOutput`，或调整 `from_stage_output()` 的输出复制逻辑时。 +- 强制：对照真实 `RequestOutput.__init__` 设置的全部公开属性，在 `OmniRequestOutput` 中以匹配的类型和默认值声明字段，并同步加入 `_REQUEST_OUTPUT_CONTENT_ATTRS`，确保 `from_stage_output()` 复制非空值。 +- 禁止：只声明新字段而遗漏复制列表，或用手写的静态字段清单代替真实 `RequestOutput` 属性 parity 检查，导致 serving 所需的 connector metadata 或 cache accounting 静默丢失。 +- 验收：用真实 `RequestOutput` 与 `OmniRequestOutput` 做属性 parity 测试；覆盖新字段默认值为 `None`，以及 `ec_transfer_params`、`num_cache_creation_tokens` 等非空值经 `from_stage_output()` 后保持不变。 ^[PR #6152] + ### EXEC-8a — DiT Euler 采样循环必须只计算一次不变条件 - 触发：修改 GLM-TTS flow-matching DiT 的 Euler 采样循环、文本 embedding、RoPE、block-causal mask、CFG batch，或新增预计算输入参数。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d4cb58587..b0f236012 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7b76b644` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e57574ae` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d926c3cc0..59af316a4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7b76b644` 验证存在 +- 源码校验：以上路径均已在 `main @ e57574ae` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 480a67b2a..4723b41e3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7b76b644`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e57574ae`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `a70d4d4b` 本轮新增 Diffusion 执行边界规则与 MiniMax H3 严格 Ulysses 局部 embedding/compact gather 规则，并登记 PR #6173 与 TeaCache extractor 来源。不得据此推断 21 倍端到端加速、其他模型或其他并行拓扑的支持；性能和质量结论仅绑定 PR 报告的精确 workload。 +本次相对 `7b76b644` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from a70d4d4b \ - --to 7b76b644 \ + --from 7b76b644 \ + --to e57574ae \ --repo \ --mode report-only ``` From dee9f677bded683b3cd8660472f496fe91fb92c7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:17:16 +0800 Subject: [PATCH 166/465] knowledge(vllm-omni): sync f39b8684 (PR #5963) Upstream-Commit: f39b8684eb6f463dcd48f3c24d88be78108dbe09 Upstream-PR: #5963 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 071386191..a1813cf9f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7b76b6446d29adc23f563b75f604159ab11ecac8 + previous_audited_sha: e57574ae290b42a1a341c86f0d1cecdb563e3187 audited_ref: main - audited_sha: e57574ae290b42a1a341c86f0d1cecdb563e3187 + audited_sha: f39b8684eb6f463dcd48f3c24d88be78108dbe09 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 34361d689..9b2d06bcf 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e57574ae290b42a1a341c86f0d1cecdb563e3187`. The canonical machine baseline +f39b8684eb6f463dcd48f3c24d88be78108dbe09`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7952fea8f..eb5bd355d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e57574ae` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f39b8684` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 4cf80e24d..25dd4de8f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e57574ae` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f39b8684` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6273c5ede..c1e429dc1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e57574ae` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f39b8684` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f88c23560..69cb55a4b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e57574ae` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f39b8684` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b0f236012..e2cd36d47 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e57574ae` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f39b8684` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 59af316a4..a9fd897a9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e57574ae` 验证存在 +- 源码校验：以上路径均已在 `main @ f39b8684` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4723b41e3..614a3d84d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e57574ae`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f39b8684`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `7b76b644` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `e57574ae` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 7b76b644 \ - --to e57574ae \ + --from e57574ae \ + --to f39b8684 \ --repo \ --mode report-only ``` From 6cf8ecf663cad685c2abd29f02ec4b67b3965d26 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:23:46 +0800 Subject: [PATCH 167/465] knowledge(vllm-omni): sync ce6ab0ca (PR #6070) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: ce6ab0ca4666350329261adaf0ce6198764ca7b2 Upstream-PR: #6070 Knowledge-Delta: 新增 LTX-2.5 的共享 attention、精度、SP padding、checkpoint revision/profile、Cache-DiT、model_extras 与 serving resize 合同，并补充现有 LTX phase LoRA 规则的 PR 证据。 --- adapters/vllm_omni/release_baseline.yaml | 8 ++++---- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-attention.md | 9 ++++++++- .../diffusion/rules-component-lifecycle.md | 12 ++++++++++-- .../vllm-omni/components/diffusion/rules-lora.md | 2 +- .../components/diffusion/rules-sp-padding.md | 9 ++++++++- .../components/diffusion/rules-tensor-dtype.md | 12 ++++++++++-- .../vllm-omni/components/diffusion/rules.md | 16 +++++++++++++++- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-image-task-envelope.md | 10 +++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 17 files changed, 84 insertions(+), 25 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a1813cf9f..1e6a5dbf7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e57574ae290b42a1a341c86f0d1cecdb563e3187 + previous_audited_sha: f39b8684eb6f463dcd48f3c24d88be78108dbe09 audited_ref: main - audited_sha: f39b8684eb6f463dcd48f3c24d88be78108dbe09 + audited_sha: ce6ab0ca4666350329261adaf0ce6198764ca7b2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 86 sha256: 039140fb0a8e95f13015d11af550db8ed16e6071448889f243c1525de5dff493 diffusion: - count: 61 - sha256: f751159bd0647552d405a0481ad582caf8f928787b8d5c7863fe5e863e12ca08 + count: 63 + sha256: c776968386eba1c3456afa3ffe518c8e960880ed583b10e71d3f53b21ffe9cbb pipelines: count: 57 sha256: 1cfc25b2c2bb9a543a77830477a795ec59e44d4abd3d07c9b8f6f05ec07d1aac diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9b2d06bcf..d6e65d5e0 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f39b8684eb6f463dcd48f3c24d88be78108dbe09`. The canonical machine baseline +ce6ab0ca4666350329261adaf0ce6198764ca7b2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index eb5bd355d..75634791f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f39b8684` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ce6ab0ca` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 25dd4de8f..514d7650e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f39b8684` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ce6ab0ca` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index df880bb46..c42ec55d1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py"] +sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py"] confidence: high --- @@ -84,3 +84,10 @@ confidence: high - 禁止：因为 cross-attention 的 Q/K 长度不同就丢弃 mask；把只匹配 query 长度的二维 mask 交给 Flash varlen；把 backend 的 `supports_attention_mask` 能力等同于支持任意 mask layout。 - 验收：CPU/mock 覆盖 masked self-attention、masked unequal-length cross-attention 与 unmasked path，分别断言 SDPA/native dispatch 和 mask shape；真实 SP 再以带 padding 的 LTX 输出对照未分片参考。^[PR #5500] +## DIFF-1n — cuDNN attention 的单 token 形状必须提前走 MATH + +- 触发：修改 cuDNN diffusion attention dispatch，或引入可能出现单 token 的动态 Q/K 序列长度。 +- 强制：当 query 或 key 的序列长度为 1 时，在进入 cuDNN-only context 前显式使用 `sdpa_kernel([SDPBackend.MATH])`；其他形状才使用显式 cuDNN，并保留符号 shape 被 cuDNN 拒绝时的运行时 fallback。 +- 禁止：把单 token 形状交给 cuDNN 后等待 eager exception；用包含 FLASH/MATH 的 priority list 掩盖 backend 选择；因 Q/K 长度不同而丢弃已有 attention mask。 +- 验收：CPU/mock 参数化覆盖 `(1,1)`、`(8,1)`、`(1,8)` 与 `(8,8)`，精确断言选择的 SDP backend；另覆盖 compile、masked unequal-length 与 unmasked native path，并说明真实 GPU 数值仍需独立验证。^[PR #6070] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md index b025068d4..f4dd71959 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md @@ -1,10 +1,10 @@ --- title: "Diffusion component lifecycle 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5720", "PR #5853", "PR #5884", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py] +sources: ["PR #5720", "PR #5853", "PR #5884", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py"] confidence: high --- @@ -79,3 +79,11 @@ residency 留在模型 owner。规则入口与其他共享机制仍见 [Diffusio hint 没进入 key，但 H3 `forward` 明确拒绝 `len(prompts) != 1`，不得把该例外推广到 batched adopter。 - 验收：两套 scheduler 都覆盖 same intent co-batch、accelerated/reference 分离、omitted/explicit 分离； 每个新 policy input 用 mixed batch 证明 key 完整，或测试 single-request guard。^[PR #5853] + +### DIFF-2r — LTX Cache-DiT 支持必须由 recipe 显式限定 + +- 触发：LTX 修改 Cache-DiT adapter、CFG guidance batching 或 phase recipe 的能力声明。 +- 强制：Cache-DiT 能力由 recipe 的 `supports_cache_dit` 显式声明；LTX guidance 将所有 guidance pass 融合进 batch 并每个 denoise step 只调用一次 Transformer，因此 `has_separate_cfg=False`；当前只有明确声明的 LTX-2/2.3 one-stage 与 positive-only recipe 可启用，LTX-2.5 和多阶段 recipe 必须在组件初始化前拒绝。 +- 禁止：从 one-stage/multi-stage 名称推断 Cache-DiT 支持；沿用 separate-CFG step counter；把 LTX-2.5 distilled/full 或两阶段 schedule 当成已 qualified 的 Cache-DiT 路径。 +- 验收：参数化检查每个 LTX recipe 与 adapter flag，覆盖启用、禁用和 component-init 前 rejection，并验证 fused guidance 每个 denoise step 只推进一次；无真实硬件 Cache-DiT accuracy 证据时不得扩大支持范围。^[PR #6070] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md index e023bcaa2..15ebb932d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py"] +sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md b/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md index 9be57229a..ebc0bdb48 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-sp-padding.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4645", docs/user_guide/diffusion/parallelism/sequence_parallel.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/sp_plan.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/hooks/sequence_parallel.py, vllm_omni/diffusion/models/flux2/flux2_transformer.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_latents.py", "vllm_omni/diffusion/models/ltx2/ltx2_denoise.py", "vllm_omni/diffusion/models/ltx2/ltx2_guidance.py"] +sources: ["PR #4645", docs/user_guide/diffusion/parallelism/sequence_parallel.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/sp_plan.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/hooks/sequence_parallel.py, vllm_omni/diffusion/models/flux2/flux2_transformer.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_latents.py", "vllm_omni/diffusion/models/ltx2/ltx2_denoise.py", "vllm_omni/diffusion/models/ltx2/ltx2_guidance.py", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_runtime.py"] confidence: high --- @@ -57,3 +57,10 @@ confidence: high - 禁止：把零 padding 当作 sampler state、让 padding 消耗 request RNG、用 padding 参与 guidance rescale，或在 Ring 无法消费 key mask 时继续执行 unmasked attention。 - 验收：覆盖 SP=1、非整除 padding、provided/generated latents、RNG 后续状态、零尾部、rescale token count、masked SDPA dispatch 和 Ring rejection；有效 audio/video 输出须与无 padding 的固定 seed 参考保持既定容差。^[PR #5500] +## SPPAD-2c — audio padding 必须在 sampler 更新后退出逻辑状态 + +- 触发：LTX AV pipeline 在 SP 下处理非整除的 audio latent length，或修改 scheduler/ancestral sampler 后的 audio state。 +- 强制：只让逻辑 audio token 进入 RNG、guidance rescale 和 sampler；每次 scheduler update 后立即清零 SP padding tail，并在所有 phase、terminal step 和 ancestral path 保持原始有效帧数。 +- 禁止：把 padding 当作 sampler state、让 padding 消耗请求 RNG、用 padded length 计算 rescale，或在 padding 仍可能被 attention 消费时继续执行 unmasked path。 +- 验收：覆盖 SP=1、partial padding、provided/generated latents、Euler/ancestral、terminal step 和后续 RNG 状态；断言有效 audio 与无 padding 固定 seed reference 一致，padding 始终为零。^[PR #6070] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-tensor-dtype.md b/knowledge/repos/vllm-omni/components/diffusion/rules-tensor-dtype.md index 466e29004..983e88e7f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-tensor-dtype.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-tensor-dtype.md @@ -1,10 +1,10 @@ --- title: "Diffusion tensor dtype 规则" created: 2026-09-03 -updated: 2026-09-03 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5067", "PR #5068", "PR #5174", "PR #5981", "Issue #5880", vllm_omni/diffusion/layers/norm.py, tests/diffusion/layers/test_norm.py] +sources: ["PR #5067", "PR #5068", "PR #5174", "PR #5981", "Issue #5880", vllm_omni/diffusion/layers/norm.py, tests/diffusion/layers/test_norm.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_latents.py"] confidence: high --- @@ -28,3 +28,11 @@ confidence: high - 证据边界：相同输入的 online/offline 与 retry 重复得到相同失败值，只支持“确定性数值漂移”； 未绑定软件 commit、硬件与可复核 artifact 的截图不能建立跨硬件或 bit-exact parity。 ^[PR #5067] ^[PR #5068] ^[PR #5174] ^[PR #5981] + +## DIFF-1o — LTX latent layout 与混合精度边界必须保持官方顺序 + +- 触发：修改 LTX-2/2.3/2.5 的 latent、guidance、sigma 或 ancestral sampler，并涉及 BF16/FP32 混合计算和请求级 generator。 +- 强制：生成和提供的 video latent 都保持官方 token-major storage layout；noising/conditioning 在 FP32 中按官方 lerp 顺序计算后再 materialize 为模型 dtype；velocity 使用 host scalar sigma，ancestral extra-noise 使用独立的 `seed + 10000` 流，并保持请求 generator 所有权。 +- 禁止：只对 LTX-2.5 应用 token-major layout；在 BF16 mask 上提前乘 `noise_scale`；直接用 CUDA scalar tensor 做 sigma 除法；让 padding 或共享全局 RNG 改变请求的随机状态；仅凭 shape 和可运行性宣称数值 parity。 +- 验收：对 2、2.3、2.5 参数化 seeded generated/provided latent 测试，逐值核对 layout、stride、后续 RNG 状态、FP32 noising 和 sigma arithmetic；ancestral step 再与固定官方 reference 对照，硬件 parity 必须固定 attention backend 与 checkpoint revision。^[PR #6070] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 3da044f9f..5728e3ad4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py"] confidence: high --- @@ -115,6 +115,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：仅凭 SP rank/world size 推断局部 embedding 安全；让 Ring、AllGather-KV、advanced UAA、缺少 hook、非 forward context 或不可整除长度误走局部路径；把模型专属的局部 row 重建合同泛化为所有 diffusion 模型。 - 验收：覆盖 strict hook 的 rank span、缺 hook、forward context 不可用、SP=1、advanced UAA、Ring、AllGather-KV 和非整除长度 fallback，并回归 TeaCache extractor 的完整 span；当前合同只由 MiniMax-H3 caller 证明，其他模型仍需独立验证。 ^[PR #6173] +### DIFF-1p — LTX-2.5 I2V 必须先执行 CRF conditioning 再 resize + +- 触发：LTX-2.5 I2V 修改 reference-image conditioning、`image_crf`、resize 顺序或 PyAV/FFmpeg 依赖。 +- 强制：PIL 输入默认执行 CRF 18，再进行 aspect-preserving resize；非零 `image_crf` 只接受 PIL 图像，tensor 输入必须显式使用 `image_crf=0` opt out；缺少 PyAV/libx264 要返回明确错误，任一尺寸小于 2 时在编码前原样返回。 +- 禁止：在 CRF round trip 前 resize 或 squash 图像；按模型路径名猜版本；把非零 CRF 静默应用到 tensor；将 1-pixel 维度向下取整为零后交给 libx264。 +- 验收：覆盖默认/显式 CRF、PIL/tensor、无 PyAV、缺少 libx264、1xN 图像和非等比例输入，断言 CRF 顺序、错误类型、原图保护以及最终 conditioning shape。^[PR #6070] + ### DIFF-2a — checkpoint remap 必须追到已注册且真实消费的目标 - 触发：增加或修改 weight mapper、scale 名称、quantization adapter 或 key resolution。 @@ -225,6 +232,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：只检查 legacy `Fp8LinearMethod`、把非连续 transpose view 直接交给 `fully_shard`，或因新版 online method 已被识别就宣称所有 FP8 method 和布局均兼容。 - 验收：legacy 与 `Fp8PerTensorOnlineLinearMethod` 都覆盖 regression test，断言重写层数、weight contiguous、`(out, in)` shape，以及 kernel 返回的转置 view shape/stride；另按 HSDP/FSDP 规则执行真实 `fully_shard` 验证。^[PR #5677] +### DIFF-2q — LTX checkpoint profile 必须贯穿 revision 与 transformer 子目录 + +- 触发：新增 LTX checkpoint/version profile、Hub `revision`、Full/Distilled transformer 配置或组件加载路径。 +- 强制：显式 `model_version` 优先于结构启发式；`revision` 必须贯穿 metadata detection、prefetch、tokenizer、全部组件、transformer config/weight source、scheduler 和 post-process sample-rate；LTX-2.5 Full/SFT 使用 `transformer_full`，distilled 与旧版本使用 `transformer`，并在缺少 Gemma4 时提示 `transformers>=5.10.1,<5.15`。 +- 禁止：按路径名识别版本；只给 config lookup 传 revision 而让权重加载回到 HEAD；用 `transformer/config.json` 解释 Full checkpoint；把 transient metadata/Hub 异常静默降级为 LTX-2 并宣称 profile 已确定。 +- 验收：固定 revision 的 converted/official split 链路逐项断言所有 loader 调用和输出 metadata；用两个 transformer config marker 验证 Full 子目录选择，覆盖 local/HF、legacy/distilled/Full profile 及缺失 encoder 版本错误。^[PR #6070] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c1e429dc1..6fa64fb5c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f39b8684` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ce6ab0ca` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 69cb55a4b..16d7bf6b2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f39b8684` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ce6ab0ca` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md index 01f18701e..f931cfb0d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #6049", "PR #6076", vllm_omni/model_extras/] +sources: ["PR #6049", "PR #6076", vllm_omni/model_extras/, "PR #6070", "vllm_omni/model_extras/registry.py"] confidence: high --- @@ -33,3 +33,11 @@ confidence: high - 验收：registry 测试断言目标模型的 extra-body 参数和 `zero_to_one` 范围、普通 pipeline 的默认范围；覆盖单帧与 list-valued 视频的明确范围、混合正负值的统一转换，以及最终视频导出结果。 ^[PR #6076] 相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 + +## EXEC-6c — model_extras resolver 必须承载模型专有版本策略 + +- 触发：`model_extras` 增加模型版本相关的 request、transformer-config 或 media-processing resolver。 +- 强制：按解析后的 `model_class_name` 在 registry 声明 resolver；公开 accessor 必须把真实 `model` 与 `revision` 传给 model-owned hook；共享 config/serving consumer 只调用 accessor，unknown model 保持安全默认值。 +- 禁止：在共享 serving/data 代码中写 LTX class/version 分支；按 repository basename 猜版本；用 class 名代替 checkpoint metadata；丢弃 pinned revision 或把共享 `LTX2Pipeline` 的策略无条件应用于 2、2.3、2.5。 +- 验收：覆盖 arbitrary local path、2/2.3/2.5 metadata、Full/distilled pipeline class、unknown class 和 pinned revision；断言 extra-body 声明、transformer subfolder 与 reference-image policy 都经 registry accessor 到达真实 consumer。^[PR #6070] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e2cd36d47..7dbb5bd89 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f39b8684` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ce6ab0ca` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a9fd897a9..665762a0f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f39b8684` 验证存在 +- 源码校验：以上路径均已在 `main @ ce6ab0ca` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index d13ce39dd..1aafcaad6 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070"] confidence: high --- @@ -126,6 +126,13 @@ confidence: high - 禁止：要求没有可用 speaker 的模型提供一个实际存在的 voice 名称；让 voice 列表遗漏 `default` 或上传 speaker；把 `default` 占位值传入后端作为真实 speaker；仅支持字符串而拒绝合法 VoiceID 对象。 - 验收：覆盖无上传 speaker 时的字符串和 `{\"id\": ...}` 请求、未知 voice 的 400、上传前后 `default` 与新增 speaker 的列表和语音请求，以及注册并删除同名 `default` 后占位行为恢复；另需验证同名上传 voice 的实际使用，而不只断言请求成功。 ^[PR #5878] +## SERV-1h — I2V resize ownership 必须按 checkpoint version 解析 + +- 触发：在线视频 API 处理 I2V reference image，且 pipeline 的预处理可能随 checkpoint version 改变。 +- 强制：serving 从 active diffusion `od_config` 取得 `model_class_name`、`model` 和 `revision`，通过 model_extras resolver 判断 resize ownership；只有 LTX-2.5 保留 source geometry 交给 pipeline，LTX-2/2.3 继续在 serving 侧 resize，随后由 pipeline 执行其版本化 conditioning。 +- 禁止：以共享 `LTX2Pipeline` class 全局开启 preserve-image-size；按请求目标尺寸或路径名猜版本；跳过 revision；未经版本化回归就让 online 与 legacy offline I2V framing 政策改变。 +- 验收：用 2、2.3、2.5 checkpoint metadata、Full/Distilled class、pinned revision 和 off-aspect image 覆盖 HTTP I2V，断言 legacy resize、2.5 source geometry、CRF 委托及最终尺寸各自符合合同。^[PR #6070] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 614a3d84d..ab9bd2953 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f39b8684`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ce6ab0ca`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `e57574ae` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `f39b8684` 本轮沉淀了 PR #6070 的共享 cuDNN 单 token fallback、LTX BF16 latent/RNG 精度边界、SP audio padding 清理、LTX checkpoint revision/profile 加载合同、Cache-DiT recipe 能力门禁、model_extras resolver seam，以及在线 I2V 的版本化 resize/CRF 委托。未把 CI、文档、recipe、测试 harness、别名注册和依赖变更单独写成规则；官方 parity 数字仍只代表固定 revision、BF16 cuDNN 和指定硬件，不能推断为跨硬件或所有 backend 的通用 parity。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from e57574ae \ - --to f39b8684 \ + --from f39b8684 \ + --to ce6ab0ca \ --repo \ --mode report-only ``` From 16afb25852a04773d1eb9673ff9f699a607220fe Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:25:45 +0800 Subject: [PATCH 168/465] knowledge(vllm-omni): sync 136cb27b (PR #6023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 136cb27b05385cf2ce7246e6815154ee86525d0f Upstream-PR: #6023 Knowledge-Delta: 补充 diffusion request-mode async output 的 storage-aware IPC、批量拆分与 future 原子交付、result queue 串行写入及 sleep 前 drain 合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-output-lifecycle.md | 12 ++++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1e6a5dbf7..c2dd15035 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f39b8684eb6f463dcd48f3c24d88be78108dbe09 + previous_audited_sha: ce6ab0ca4666350329261adaf0ce6198764ca7b2 audited_ref: main - audited_sha: ce6ab0ca4666350329261adaf0ce6198764ca7b2 + audited_sha: 136cb27b05385cf2ce7246e6815154ee86525d0f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d6e65d5e0..379ebc890 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ce6ab0ca4666350329261adaf0ce6198764ca7b2`. The canonical machine baseline +136cb27b05385cf2ce7246e6815154ee86525d0f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 75634791f..f61c4b2bd 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ce6ab0ca` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 136cb27b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 514d7650e..3e1aa75ae 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ce6ab0ca` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 136cb27b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 0e5bb2cad..d5ec407ed 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -1,10 +1,10 @@ --- title: "Diffusion output 与 multiprocess runtime 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023"] confidence: high --- @@ -90,3 +90,11 @@ confidence: high content、唇音同步或完整 A/V alignment。 - 验收：目标 focused test 解 mux 后断言第一个有 PTS 的 AAC packet `PTS<0`； 样本和 A/V 对齐结论仍需独立的端到端波形/时序验收。^[PR #5978] + +## DIFF-1q — request-mode async output 交付必须覆盖传输、竞态与生命周期 + +- 触发：修改 request-mode diffusion 的 async D2H/SHM output、`MessageQueue` 写入、batch split、result pump、future bookkeeping，或 worker 的 sleep/内存释放生命周期。 +- 强制：per-request tensor view 的 IPC packing 必须按 `max(view_bytes, storage_bytes)` 判断并只把 view 内容放入 SHM；`execute_batch` 注册 split map 时必须在同一锁下接管早到的 batch output；`result_mq` 的 `COMPUTE_DONE` 与 `OUTPUT_READY` 写入必须串行化；result pump 必须先完成 SHM unpack，再在 `_futures_lock` 下原子 resolve waiter 或缓存 completed output；worker 执行 `sleep`/`handle_sleep_task` 和本地 sleep task 前必须等待 `drain_async_outputs()`，并在 `_ASYNC_OUTPUT_TIMEOUT` 超时日志中输出 executor bookkeeping。 +- 禁止：按 view 大小判断而让共享 batch storage 被 pickle 重复发送；在 split map 建立前缓存一个无人消费的 batch-level output；让多个线程并发写单写者 `MessageQueue`；SHM unpack 期间释放 future 锁导致 waiter orphan；释放 device memory 时仍允许后台 D2H/SHM 线程读取源 tensor；把单 tensor storage-aware packing 当作 aggregate serialized message 已不会溢出的证明，也不得据此推断外部 HTTP 视频传输已优化。 +- 验收：覆盖 per-request shared-storage wire-size regression、早到和晚到的 batch `OUTPUT_READY`、已有 waiter 与缓存 future 的原子 resolution、并发 result queue writer 无 overlap、pending output 的 drain/timeout 及 sleep-before-drain 顺序；同时验证 non-sleep RPC 不 drain、SHM unpack/error path、超时诊断状态和 worker shutdown cleanup。^[PR #6023] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6fa64fb5c..51d93f23f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ce6ab0ca` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 136cb27b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 16d7bf6b2..c91562d3e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ce6ab0ca` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 136cb27b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7dbb5bd89..0f9e719bf 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ce6ab0ca` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 136cb27b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 665762a0f..9209dfe29 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ce6ab0ca` 验证存在 +- 源码校验：以上路径均已在 `main @ 136cb27b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ab9bd2953..55b140599 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ce6ab0ca`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 136cb27b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `f39b8684` 本轮沉淀了 PR #6070 的共享 cuDNN 单 token fallback、LTX BF16 latent/RNG 精度边界、SP audio padding 清理、LTX checkpoint revision/profile 加载合同、Cache-DiT recipe 能力门禁、model_extras resolver seam，以及在线 I2V 的版本化 resize/CRF 委托。未把 CI、文档、recipe、测试 harness、别名注册和依赖变更单独写成规则；官方 parity 数字仍只代表固定 revision、BF16 cuDNN 和指定硬件，不能推断为跨硬件或所有 backend 的通用 parity。 +本次相对 `ce6ab0ca` 本轮将 PR #6023 的 diffusion request-mode async output 可靠性交付约束写入 `rules-output-lifecycle.md`，覆盖 storage-aware IPC packing、batch split/future 竞态、result queue 单写者以及 sleep 前 drain。现有 `_index.md` 已登记该规则页，因此无需新增索引行。该规则不表示 aggregate oversized message 已完全解决，也不表示 HTTP 视频传输或取消 future 问题已修复；docs、tests 和 tooling 变更按 work-topic 路由丢弃。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from f39b8684 \ - --to ce6ab0ca \ + --from ce6ab0ca \ + --to 136cb27b \ --repo \ --mode report-only ``` From f44a1390158e4f678665c543816eb09e077c2ed0 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:26:18 +0800 Subject: [PATCH 169/465] knowledge(vllm-omni): sync 555336e8 (PR #6032) Upstream-Commit: 555336e887da035aa473754d3f87ca7d8eb878a4 Upstream-PR: #6032 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c2dd15035..376552ef9 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ce6ab0ca4666350329261adaf0ce6198764ca7b2 + previous_audited_sha: 136cb27b05385cf2ce7246e6815154ee86525d0f audited_ref: main - audited_sha: 136cb27b05385cf2ce7246e6815154ee86525d0f + audited_sha: 555336e887da035aa473754d3f87ca7d8eb878a4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 379ebc890..88a37f97b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -136cb27b05385cf2ce7246e6815154ee86525d0f`. The canonical machine baseline +555336e887da035aa473754d3f87ca7d8eb878a4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f61c4b2bd..9d2df073f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 136cb27b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 555336e8` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3e1aa75ae..8c5de4ee0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 136cb27b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 555336e8` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 51d93f23f..8b77b661d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 136cb27b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 555336e8` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c91562d3e..8895383b1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 136cb27b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 555336e8` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0f9e719bf..6998ee968 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 136cb27b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 555336e8` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9209dfe29..1dd358240 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 136cb27b` 验证存在 +- 源码校验：以上路径均已在 `main @ 555336e8` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 55b140599..259451a98 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 136cb27b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 555336e8`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `ce6ab0ca` 本轮将 PR #6023 的 diffusion request-mode async output 可靠性交付约束写入 `rules-output-lifecycle.md`，覆盖 storage-aware IPC packing、batch split/future 竞态、result queue 单写者以及 sleep 前 drain。现有 `_index.md` 已登记该规则页，因此无需新增索引行。该规则不表示 aggregate oversized message 已完全解决，也不表示 HTTP 视频传输或取消 future 问题已修复；docs、tests 和 tooling 变更按 work-topic 路由丢弃。 +本次相对 `136cb27b` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from ce6ab0ca \ - --to 136cb27b \ + --from 136cb27b \ + --to 555336e8 \ --repo \ --mode report-only ``` From 97a651ee9937fa1ddf654fa937448cc7d49c13e0 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:27:45 +0800 Subject: [PATCH 170/465] knowledge(vllm-omni): sync 09b6a160 (PR #6214) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 Upstream-PR: #6214 Knowledge-Delta: 新增 SERV-8a：本地 stage launch 必须在限定 scope 内应用 runtime.env，并在 launch 后恢复父环境。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 376552ef9..df0d688d7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 136cb27b05385cf2ce7246e6815154ee86525d0f + previous_audited_sha: 555336e887da035aa473754d3f87ca7d8eb878a4 audited_ref: main - audited_sha: 555336e887da035aa473754d3f87ca7d8eb878a4 + audited_sha: 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 88a37f97b..86fee9092 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -555336e887da035aa473754d3f87ca7d8eb878a4`. The canonical machine baseline +09b6a1604a3f7f32c8592cd6ce115c4b89610e92`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9d2df073f..bb4ad4274 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 555336e8` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 09b6a160` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8c5de4ee0..b69755eaa 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 555336e8` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 09b6a160` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8b77b661d..883364733 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 555336e8` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 09b6a160` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8895383b1..75ff044e2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 555336e8` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 09b6a160` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6998ee968..32511b512 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 555336e8` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 09b6a160` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1dd358240..57c7f1a89 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 555336e8` 验证存在 +- 源码校验：以上路径均已在 `main @ 09b6a160` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 1024f096d..896f6ac16 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py"] confidence: high --- @@ -147,3 +147,10 @@ confidence: high - 禁止：模型 duration scaling 后再次 resample，或以 HTTP 支持外推 WebSocket parity。 - 验收：覆盖 token/chat/embed、native/generic、三种 HTTP 输出与 WebSocket。落后后端只用显式 capability/version shim 并保留 eager fallback，不能让 CUDA 新参数成为全平台前提。^[PR #5957] + +### SERV-8a — 本地 stage launch 必须临时应用 runtime.env + +- 触发：修改 `StageRuntime` 的本地 LLM/diffusion replica launch、`runtime_cfg.env`、stage device scope 或 launch lock。 +- 强制：本地 LLM launch 必须在现有串行 launch lock 内进入 `stage_runtime_env(stage_id, runtime_cfg)`；diffusion launch 必须将其与 device scope 组合，并保持 `runtime.devices` 作为有效设备选择。scope 只覆盖子进程构造/launch，退出时恢复父进程原值，并删除此前不存在的 key。 +- 禁止：只解析或记录 `runtime.env` 却不应用到本地 launch；把 stage 环境永久污染到父进程；用 runtime env scope 替代 diffusion device scope；将该本地 scope 推广到 remote replica 或 headless worker launcher。 +- 验收：CPU/mock 回归分别覆盖 LLM 和 diffusion launch 看到 stage 值、父环境已有值恢复、原先未设置的 key 在异常后被清理，以及 diffusion launch 同时看到 stage device 值；确认 LLM launch lock 仍覆盖进程创建。^[PR #6214] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 259451a98..5b45377c5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 555336e8`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 09b6a160`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `136cb27b` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `555336e8` 本轮仅沉淀 Serving 本地 stage launch 的 runtime.env 生命周期规则；测试文件按 ci/testing 路由未写入，不能据此推断 remote replica 或 headless worker 环境行为发生变化。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 136cb27b \ - --to 555336e8 \ + --from 555336e8 \ + --to 09b6a160 \ --repo \ --mode report-only ``` From 125ea912cbcd8e8667e7f228e49ff4cdc92a45fe Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:31:00 +0800 Subject: [PATCH 171/465] knowledge(vllm-omni): sync d1e230c9 (PR #5910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: d1e230c95ba12aec7664ee6fd18c0b2b2d0d6187 Upstream-PR: #5910 Knowledge-Delta: 记录全局/组件量化解析、MiniMax-H3 text encoder 的 online FP8 安全边界，以及在线量化权重在 layerwise/DLO no-AllGather 中的流式加载与 stride 保持合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 6 +++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-loader-contract.md | 7 ++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/flux2/rules.md | 4 ++-- .../vllm-omni/models/minimax-h3/rules-deployment.md | 9 +++++++-- .../repos/vllm-omni/models/minimax-h3/rules-loading.md | 7 ++++++- 14 files changed, 39 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index df0d688d7..07f75d3ae 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 555336e887da035aa473754d3f87ca7d8eb878a4 + previous_audited_sha: 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 audited_ref: main - audited_sha: 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 + audited_sha: d1e230c95ba12aec7664ee6fd18c0b2b2d0d6187 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 86fee9092..d7b9b8b2e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -09b6a1604a3f7f32c8592cd6ce115c4b89610e92`. The canonical machine baseline +d1e230c95ba12aec7664ee6fd18c0b2b2d0d6187`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bb4ad4274..d03b7460e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 09b6a160` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d1e230c9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b69755eaa..605d87016 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 09b6a160` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d1e230c9` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 5728e3ad4..e41a4c9fc 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910"] confidence: high --- @@ -239,6 +239,10 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：按路径名识别版本；只给 config lookup 传 revision 而让权重加载回到 HEAD；用 `transformer/config.json` 解释 Full checkpoint；把 transient metadata/Hub 异常静默降级为 LTX-2 并宣称 profile 已确定。 - 验收：固定 revision 的 converted/official split 链路逐项断言所有 loader 调用和输出 metadata；用两个 transformer config marker 验证 Full 子目录选择，覆盖 local/HF、legacy/distilled/Full profile 及缺失 encoder 版本错误。^[PR #6070] +### DIFF-2s — 在线量化加载与 layerwise offload 必须保留物理权重布局 + +- 触发：修改在线/runtime quantization 与 layerwise offload、DLO 权重加载、非连续 FP8 权重 flatten/repoint 或加载后处理流程。\n- 强制：在线量化层必须在消费完对应 checkpoint weight 后才可流式移回 CPU，并跳过已完成层的重复 `process_weights_after_loading`；继续加载其他模型权重前释放可复用的量化缓存。对非连续权重按物理 storage 顺序打包，记录 `shape`、`stride` 和实际 storage span，并在普通 layerwise、DLO prefetch 与 resident replay 中用 `torch.as_strided` 重建；runtime-created FP8 与 DLO 组合必须使用 no-AllGather 路径。\n- 禁止：对转置 Cutlass FP8 weight 使用逻辑 `.flatten()` 后再 `.view(shape)`，丢失物理布局；为调用幂等处理而把已量化 CPU 层重新搬回 accelerator；把 runtime-created FP8 宣称支持 sharded DLO AllGather。\n- 验收：CPU/mock 测试验证普通 layerwise、DLO prefetch 和 resident replay 的值与 stride 均保持，覆盖不同 storage span、streaming 顺序、已处理层跳过与缓存释放；online FP8 + DLO AllGather 必须 fail fast，no-AllGather 路径须完成加载并通过对应输出/资源 smoke。^[PR #5910] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 883364733..f5f0a2b62 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 09b6a160` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d1e230c9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 75ff044e2..089a800a7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 09b6a160` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d1e230c9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md index 44762f53a..af492a358 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/] +sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910"] confidence: high --- @@ -52,3 +52,8 @@ confidence: high - 验收：对已验证 checkpoint 逐模块核对加载计数、无 missing/extra tensor 和预期 dtype；对缺失配置块、架构常量不匹配、部分 shard 及错误命名空间执行 fail-fast 测试，并确认推理前就暴露错误。^[PR #4765] 相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 + +## EXEC-2d — 全局量化配置与组件配置必须明确区分作用域 + +- 触发：修改共享 quantization factory、pipeline component 配置解析或 `quantization_config` 的全局/按组件语义。\n- 强制：普通 `QuantizationConfig` 必须原样解析到每个由 pipeline 构造且支持量化的组件；只有 `ComponentQuantizationConfig` 的显式 component map 可以缩小作用域。resolver、pipeline 和 quantizable layer 必须使用一致的 runtime prefix namespace，并仅让实际支持的 vLLM quantizable layer 消费配置。\n- 禁止：用 `hasattr(quant_config, "resolve")` 等隐式约定替代共享 resolver；因为组件属于同一模型就自动重写任意 `torch.nn` layer、embedding、norm、VAE 或其他不支持量化的模块；解析到单一组件后仍要求另一套 owner prefix。\n- 验收：CPU/mock 与真实构造测试分别覆盖 global config 同时命中 `transformer`/`text_encoder`、component map 只命中指定组件、未命中返回 `None`、最长 prefix 匹配及 unsupported layer 保持 checkpoint dtype；Flux2 与 MiniMax-H3 的 resolver 必须共同通过该合同。^[PR #5910] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 32511b512..16a63b281 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 09b6a160` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d1e230c9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 57c7f1a89..c1e20a7e4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 09b6a160` 验证存在 +- 源码校验：以上路径均已在 `main @ d1e230c9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5b45377c5..acff932a6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 09b6a160`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d1e230c9`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `555336e8` 本轮仅沉淀 Serving 本地 stage launch 的 runtime.env 生命周期规则；测试文件按 ci/testing 路由未写入，不能据此推断 remote replica 或 headless worker 环境行为发生变化。 +本次相对 `09b6a160` 本轮新增全局/组件量化解析、MiniMax-H3 文本编码器 online FP8 安全边界，以及在线量化权重在 layerwise/DLO no-AllGather 中的流式加载与 stride 保持规则。不得据此推断任意 encoder 都会自动量化、runtime FP8 支持 DLO AllGather，或将 PR 中的性能/显存数字泛化为硬件保证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 555336e8 \ - --to 09b6a160 \ + --from 09b6a160 \ + --to d1e230c9 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/flux2/rules.md b/knowledge/repos/vllm-omni/models/flux2/rules.md index e76093fce..2aac6406c 100644 --- a/knowledge/repos/vllm-omni/models/flux2/rules.md +++ b/knowledge/repos/vllm-omni/models/flux2/rules.md @@ -1,10 +1,10 @@ --- title: "FLUX.2 规则" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4645", "PR #5136", vllm_omni/diffusion/models/flux2/flux2_transformer.py] +sources: ["PR #4645", "PR #5136", vllm_omni/diffusion/models/flux2/flux2_transformer.py, "PR #5910"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 0a69fb884..1edd404b3 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -1,10 +1,10 @@ --- title: "MiniMax H3 部署与证据规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910"] confidence: high --- @@ -210,3 +210,8 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule 或可复核 quality artifact，且 PR 只有 markdown/pre-commit。review 明确 Ref2VA 未测试，因此 `MODEL=Ref2VA` 重启说明只是配置建议。重测仍须记录 exact image/SHA、rank groups、NUMA policy、 all-rank peak 与 artifact/quality。^[PR #5857] + +## MMH3-3i — H3 online FP8 的 layerwise/DLO 组合必须限界并保持 stride + +- 触发：MiniMax-H3 同时启用 online FP8 与普通 layerwise offload、distributed layerwise offload、resident layers 或 transposed Cutlass weight replay。\n- 强制：online FP8 可以与 resident/no-offload、普通 layerwise offload 以及 DLO full-weight per-rank no-AllGather 路径组合；DLO 启动必须使用 `--dlo-no-use-allgather`，保留 standard loader 生成的物理布局，并在 streaming、prefetch、resident replay 中保存和恢复真实 stride。\n- 禁止：继续把 online FP8 与所有 layerwise offload一概标记为不兼容；runtime-created FP8 进入 sharded DLO AllGather；用 `.view()` 把物理转置权重恢复成 contiguous 逻辑布局；把单次 B300 的显存或速度观察写成 H3 通用容量/性能保证。\n- 验收：H3 contract 覆盖 resident、普通 layerwise、DLO no-AllGather 与 no-offload 的启动/输出路径；online FP8 + DLO AllGather 在 loader 边界明确报错；转置权重回放逐值核对 stride，并分别记录 exact GPU、shape、steps、checkpoint 和 all-rank 资源结果。^[PR #5910] + diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md index 8dd336565..d5b208291 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py] +sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #5910"] confidence: high --- @@ -49,3 +49,8 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection 映射覆盖，但未执行真实完整加载时，不能声称 shape/dtype/full-load 已回归；本改动本身 未改变 shape/dtype handling。共享 fused-loader 门禁见 [EXEC-2b](../../components/model-executor/rules-loader-contract.md#exec-2b-fused-shard-必须按-source-完整性与布局数值闭环)。^[PR #5824] + +## MMH3-1k — H3 global FP8 必须覆盖可量化文本解码器并拒绝预量化 encoder 配置 + +- 触发：修改 MiniMax-H3 的 `quantization="fp8"`、`diffusion-quantization-config`、Qwen3-VL text encoder linear 构造或其 checkpoint quantization safety filter。\n- 强制：H3 的 plain/global FP8 配置同时提供给 DiT 与已暴露为 `LinearBase` 的 Qwen3-VL text decoder；显式 `transformer`/`text_encoder` component map 只量化指定组件。eligible attention/MLP linears 使用 online FP8，vision tower、embedding、norm、RoPE、VAE 及模型指定 FP32 projection 保持 checkpoint precision；text encoder 仅允许 online FP8，BF16 encoder 必须剥离不兼容的 serialized ModelOpt/pre-quantized 配置。\n- 禁止：把 H3 global FP8 描述成 DiT-only，或据此自动量化任意普通 `torch.nn` encoder；把 serialized FP8/ModelOpt scale 配置直接交给没有对应 checkpoint 参数的 BF16 text encoder；为 H3 另造一套绕过 vLLM factory 的 FP8 method。\n- 验收：覆盖 global、DiT-only、text-encoder-only 与组合配置，断言各组件实际 quant method、未量化边界和 runtime prefix；serialized FP8 必须拒绝，ModelOpt config 必须从 BF16 text encoder 移除而保留 online FP8；text encoder fused shard/load contract 继续通过。^[PR #5910] + From 664d98ec431e12a7b5bd870071e13db27ac002f0 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:34:27 +0800 Subject: [PATCH 172/465] knowledge(vllm-omni): sync 2a315e1a (PR #5676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 2a315e1a2d3ef87fca690f7c58e00a1d35a047d0 Upstream-PR: #5676 Knowledge-Delta: 新增 Wan2.2 request-level batching 的结构兼容键、请求隔离与数值等价边界，并记录 diffusion stage batch capacity 的双启动路径传播要求。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 8 +++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 28 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 07f75d3ae..325023459 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 + previous_audited_sha: d1e230c95ba12aec7664ee6fd18c0b2b2d0d6187 audited_ref: main - audited_sha: d1e230c95ba12aec7664ee6fd18c0b2b2d0d6187 + audited_sha: 2a315e1a2d3ef87fca690f7c58e00a1d35a047d0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d7b9b8b2e..8f17d48ed 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d1e230c95ba12aec7664ee6fd18c0b2b2d0d6187`. The canonical machine baseline +2a315e1a2d3ef87fca690f7c58e00a1d35a047d0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d03b7460e..366487611 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d1e230c9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2a315e1a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 605d87016..0c6e30269 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d1e230c9` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 2a315e1a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index e41a4c9fc..57c488492 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676"] confidence: high --- @@ -278,3 +278,9 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：对已消费 marker 的 multipart 使用有限超时；超时后重新 `dequeue` 并把后续 marker 或 inline 消息与迟到 payload 配对；仅把有限超时调大作为修复。 - 验收：单测断言接收调用使用 `indefinite=True`，并让 overflow payload 延迟超过任何有限超时后再发送 inline 消息，断言 worker 按 `overflow-1`、`inline-2` 顺序处理。当前 PR 的 0.1 秒延迟仍在旧 1 秒超时内，且 monkeypatch 的模块变量未被生产代码读取，不能单独证明旧实现会失败。^[PR #4845] +### DIFF-6a — request-level batching 必须由结构兼容键和请求隔离共同约束 + +- 触发：diffusion pipeline 宣布支持 request-level batching，或 scheduler/admission 增加模型条件、`extra_args` 结构与 request-level output split。 +- 强制：预处理必须为影响物理 batch 兼容性的 pipeline condition 写入 `batch_compatibility_key`；`RequestScheduler` 必须将显式 `sample_solver` 规范化为去空格小写、将 `flow_shift` 转为 `float`，并保留未指定值为 `None`，再与现有 batch-wide sampling key 一起决定 admission。pipeline 只能批处理兼容 key 相同的请求，按 `num_outputs_per_prompt` 合并 prompt/embedding、latents、request-local generators 和条件，并拆回每个原始请求一个 `DiffusionOutput`；S2V 等 request-local metadata 不得为输出数重复物化。 +- 禁止：只按原始 `extra_args`、第一条请求或 pipeline 名称决定兼容性；把不兼容的 image/last-image、VACE 或 S2V 条件混入同一 batch；共享全局 RNG；返回无法按请求对应的聚合输出；把请求隔离和 output split 误当成不同物理 batch shape 的数值等价。 +- 验收：scheduler 测试必须区分不同 `sample_solver`、`flow_shift` 和 condition key，并证明等价显式值可归一化合批；Wan2.2 T2V/I2V/VACE/S2V 测试覆盖 generators、latents、prompt embeddings、output split 以及 image shape、last-image presence、VACE condition/reference count、S2V `init_first_frame`/`num_repeat`/audio time length/raw shape/sample rate 的拒绝路径。真实验证还必须分别检查同批请求的身份、prompt、条件、顺序和请求级 RNG 隔离；Wan2.2 BF16 的 batch-1 与 batch-2 轨迹可能不同，未经固定精度策略和 exact case 验证不得声称 bitwise 或生产级等价。^[PR #5676] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f5f0a2b62..9038a7dc9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d1e230c9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2a315e1a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 089a800a7..c8bcdef33 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d1e230c9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2a315e1a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 16a63b281..f28d0e11c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d1e230c9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2a315e1a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c1e20a7e4..86ce3e1ff 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d1e230c9` 验证存在 +- 源码校验：以上路径均已在 `main @ 2a315e1a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 896f6ac16..76646b17e 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676"] confidence: high --- @@ -154,3 +154,10 @@ confidence: high - 强制：本地 LLM launch 必须在现有串行 launch lock 内进入 `stage_runtime_env(stage_id, runtime_cfg)`；diffusion launch 必须将其与 device scope 组合，并保持 `runtime.devices` 作为有效设备选择。scope 只覆盖子进程构造/launch，退出时恢复父进程原值，并删除此前不存在的 key。 - 禁止：只解析或记录 `runtime.env` 却不应用到本地 launch；把 stage 环境永久污染到父进程；用 runtime env scope 替代 diffusion device scope；将该本地 scope 推广到 remote replica 或 headless worker launcher。 - 验收：CPU/mock 回归分别覆盖 LLM 和 diffusion launch 看到 stage 值、父环境已有值恢复、原先未设置的 key 在异常后被清理，以及 diffusion launch 同时看到 stage device 值；确认 LLM launch lock 仍覆盖进程创建。^[PR #6214] + +### SERV-9a — diffusion stage batch size 必须落到 engine admission capacity + +- 触发：diffusion stage 初始化或 replica launch 接收客户端 `batch_size`，或修改其 engine config、scheduler capacity 与 stage startup 路径。 +- 强制：`initialize_diffusion_stage` 和 `launch_diffusion_stage_replica` 在 `build_diffusion_config` 之后、创建 diffusion client/proc manager 之前，都必须将 `od_config.max_num_seqs` 设置为请求的 `batch_size`；direct、parallel 和 single-stage 路径必须使用同一 capacity 语义。 +- 禁止：只把 `batch_size` 传给 client 而保留 config 默认值；只修一个 startup 路径；从 `max_num_seqs > 1` 推断具体 pipeline 已支持 request batching，或绕过 `DIFF-6a` 的条件兼容 admission。 +- 验收：分别 mock `initialize_diffusion_stage` 与 `launch_diffusion_stage_replica`，断言构造出的 `od_config.max_num_seqs` 等于客户端 batch size，并回归 single-stage 初始化；随后以 scheduler 和 Wan2.2 pipeline 测试证明该 capacity 只开放物理槽位，兼容 key、请求隔离和输出拆分仍有效。^[PR #5676] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index acff932a6..97617db17 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d1e230c9`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2a315e1a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `09b6a160` 本轮新增全局/组件量化解析、MiniMax-H3 文本编码器 online FP8 安全边界，以及在线量化权重在 layerwise/DLO no-AllGather 中的流式加载与 stride 保持规则。不得据此推断任意 encoder 都会自动量化、runtime FP8 支持 DLO AllGather，或将 PR 中的性能/显存数字泛化为硬件保证。 +本次相对 `d1e230c9` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 09b6a160 \ - --to d1e230c9 \ + --from d1e230c9 \ + --to 2a315e1a \ --repo \ --mode report-only ``` From a546fba40547e4846b281c3271afd83ada5e11ff Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:35:01 +0800 Subject: [PATCH 173/465] knowledge(vllm-omni): sync 44448565 (PR #6120) Upstream-Commit: 444485650b19b792403b12976f1b0eeb2ac1451c Upstream-PR: #6120 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 325023459..e2c5afeca 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d1e230c95ba12aec7664ee6fd18c0b2b2d0d6187 + previous_audited_sha: 2a315e1a2d3ef87fca690f7c58e00a1d35a047d0 audited_ref: main - audited_sha: 2a315e1a2d3ef87fca690f7c58e00a1d35a047d0 + audited_sha: 444485650b19b792403b12976f1b0eeb2ac1451c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8f17d48ed..28fc3c149 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2a315e1a2d3ef87fca690f7c58e00a1d35a047d0`. The canonical machine baseline +444485650b19b792403b12976f1b0eeb2ac1451c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 366487611..d69ae757a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2a315e1a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 44448565` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0c6e30269..06d8cd5a9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2a315e1a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 44448565` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9038a7dc9..e226edd11 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2a315e1a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 44448565` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c8bcdef33..9562874d3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2a315e1a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 44448565` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f28d0e11c..77f43f7f1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2a315e1a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 44448565` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 86ce3e1ff..56c2ed921 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2a315e1a` 验证存在 +- 源码校验：以上路径均已在 `main @ 44448565` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 97617db17..a48c09966 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2a315e1a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 44448565`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `d1e230c9` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `2a315e1a` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from d1e230c9 \ - --to 2a315e1a \ + --from 2a315e1a \ + --to 44448565 \ --repo \ --mode report-only ``` From d209512a6b21ea311841403bc616a3093c226e4a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:36:12 +0800 Subject: [PATCH 174/465] knowledge(vllm-omni): sync 68bf8c2c (PR #5983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 68bf8c2c4296fb851c36173e5dcdca55ea56500c Upstream-PR: #5983 Knowledge-Delta: 新增 DIFF-1r：共享 diffusion Future 的取消竞态必须被安全吞掉，避免 result pump singleton 崩溃。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-output-lifecycle.md | 6 +++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 18 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e2c5afeca..ff1036161 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2a315e1a2d3ef87fca690f7c58e00a1d35a047d0 + previous_audited_sha: 444485650b19b792403b12976f1b0eeb2ac1451c audited_ref: main - audited_sha: 444485650b19b792403b12976f1b0eeb2ac1451c + audited_sha: 68bf8c2c4296fb851c36173e5dcdca55ea56500c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 28fc3c149..4e5d0c034 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -444485650b19b792403b12976f1b0eeb2ac1451c`. The canonical machine baseline +68bf8c2c4296fb851c36173e5dcdca55ea56500c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d69ae757a..0d9d98825 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 44448565` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 68bf8c2c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 06d8cd5a9..47aa02641 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 44448565` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 68bf8c2c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index d5ec407ed..30549aee1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023"] +sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983"] confidence: high --- @@ -98,3 +98,7 @@ confidence: high - 禁止：按 view 大小判断而让共享 batch storage 被 pickle 重复发送；在 split map 建立前缓存一个无人消费的 batch-level output；让多个线程并发写单写者 `MessageQueue`；SHM unpack 期间释放 future 锁导致 waiter orphan；释放 device memory 时仍允许后台 D2H/SHM 线程读取源 tensor；把单 tensor storage-aware packing 当作 aggregate serialized message 已不会溢出的证明，也不得据此推断外部 HTTP 视频传输已优化。 - 验收：覆盖 per-request shared-storage wire-size regression、早到和晚到的 batch `OUTPUT_READY`、已有 waiter 与缓存 future 的原子 resolution、并发 result queue writer 无 overlap、pending output 的 drain/timeout 及 sleep-before-drain 顺序；同时验证 non-sleep RPC 不 drain、SHM unpack/error path、超时诊断状态和 worker shutdown cleanup。^[PR #6023] +## DIFF-1r — 共享 Future resolve 必须对取消竞态 fail-safe + +- 触发：修改 diffusion result pump、共享 `_rpc_futures`/`_output_futures`、`asyncio.wait_for()` 超时、请求 abort 或 shutdown future cleanup。\n- 强制：共享 Future 在 `done()` 检查后仍必须通过 `try_set_result`/`try_set_exception` 解析，并在 helper 内捕获 `concurrent.futures.InvalidStateError`，丢弃取消或已完成 Future 的晚到结果/异常，使 singleton pump 继续运行；仅 freshly-created 且尚未共享的本地 Future 可直接解析。\n- 禁止：把 `not fut.done()` 与 `set_result()`/`set_exception()` 当作原子操作；在共享 Future 上保留未保护的 resolve 调用；让取消竞态异常逃逸并杀死 result pump，随后仍以健康检查通过推断任务会完成。\n- 验收：用 `done()` 固定返回 false 但实际已取消的 racy Future 覆盖 `COMPUTE_DONE`、`OUTPUT_READY`、batch split 与 shutdown cleanup，断言 pump 不抛 `InvalidStateError`、取消 Future 保持取消、同批健康 Future 仍完成，并回归相邻并发测试。\n^[PR #5983] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e226edd11..f5b2e7321 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 44448565` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 68bf8c2c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9562874d3..f30c59f08 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 44448565` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 68bf8c2c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 77f43f7f1..a2be24fd7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 44448565` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 68bf8c2c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 56c2ed921..8cf9e6fea 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 44448565` 验证存在 +- 源码校验：以上路径均已在 `main @ 68bf8c2c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a48c09966..9888a1416 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 44448565`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 68bf8c2c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `2a315e1a` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `44448565` 本轮新增共享 diffusion result pump 的取消竞态规则，记录共享 Future 晚到 resolve 不得使 singleton pump 崩溃。不得据此推断所有异步输出、取消或 shutdown 路径均已完成端到端验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 2a315e1a \ - --to 44448565 \ + --from 44448565 \ + --to 68bf8c2c \ --repo \ --mode report-only ``` From 9b565dc9d3e6f848d3006316ac0306a3fcaeb8ba Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:36:45 +0800 Subject: [PATCH 175/465] knowledge(vllm-omni): sync 9798bc3a (PR #6221) Upstream-Commit: 9798bc3a8d5a400b97fba7402f42510577e7302d Upstream-PR: #6221 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ff1036161..eac92bce4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 444485650b19b792403b12976f1b0eeb2ac1451c + previous_audited_sha: 68bf8c2c4296fb851c36173e5dcdca55ea56500c audited_ref: main - audited_sha: 68bf8c2c4296fb851c36173e5dcdca55ea56500c + audited_sha: 9798bc3a8d5a400b97fba7402f42510577e7302d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4e5d0c034..554de7ab2 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -68bf8c2c4296fb851c36173e5dcdca55ea56500c`. The canonical machine baseline +9798bc3a8d5a400b97fba7402f42510577e7302d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0d9d98825..b1ea00384 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 68bf8c2c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9798bc3a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 47aa02641..65a9b2a34 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 68bf8c2c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9798bc3a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f5b2e7321..2d9e77308 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 68bf8c2c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9798bc3a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f30c59f08..b629e9596 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 68bf8c2c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9798bc3a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a2be24fd7..f5c9cbbdf 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 68bf8c2c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9798bc3a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8cf9e6fea..3f734cdea 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 68bf8c2c` 验证存在 +- 源码校验：以上路径均已在 `main @ 9798bc3a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9888a1416..5ee8f2611 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 68bf8c2c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9798bc3a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `44448565` 本轮新增共享 diffusion result pump 的取消竞态规则，记录共享 Future 晚到 resolve 不得使 singleton pump 崩溃。不得据此推断所有异步输出、取消或 shutdown 路径均已完成端到端验证。 +本次相对 `68bf8c2c` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 44448565 \ - --to 68bf8c2c \ + --from 68bf8c2c \ + --to 9798bc3a \ --repo \ --mode report-only ``` From d2cb2ccd9d9b3fa06841a3d95000ecf874d30275 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:51:18 +0800 Subject: [PATCH 176/465] knowledge(vllm-omni): retire the misnamed catch-all section in model-executor rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third crossing of the 32768-byte gate on this page, at unit 169. The previous split bought 23 units, and the section that refilled tells the story: 跨 stage bridge 与 batch 合同 held EXEC-7a through EXEC-13a and not one bridge-batch rule. A rule that opens a new series has no sibling to anchor against, so it is appended at the end of the page -- inside whichever section happens to be last, which then becomes a catch-all whose name stops describing it. Those nine rules are three real categories, extracted as such, and the section itself is removed rather than renamed: EXEC-7a-7b -> rules-output-contract.md Omni output type, field and copy contract EXEC-8a, 9a, 11a-11b -> rules-runtime-hot-paths.md sampling-loop invariants, fixed input cache, AR audio side path, codec frame ledger EXEC-10a, 12a-13a -> rules-platform-backends.md NPU runner interface, ROCm paged attention, NPU patches Rule IDs unchanged, three group rows added, all three registered. Headroom after: rules.md 13329 B, up from 3224 at the halt. Also repaired a blank line that had split the _index.md table in half, orphaning six rows into paragraph text. I introduced it in an earlier registration: the insertion point came from a regex ending in \s*$, and \s matches newlines, so the anchor landed after the following blank line rather than at the end of the row. GFM ends a table at the first blank line, so those rows rendered as pipe-delimited prose and the navigation they were added for did not exist. A first attempt to repair it removed every blank line between two table rows, which merged the two genuinely distinct tables in diffusion, serving and configuration rules -- turning each second table's header into a data row. Reverted -- though not completely: the same pass had merged the two tables in this page too, and restoring the other four masked it until review caught it. A blank line followed by a |---| separator starts a new table and must stay. No unit is consumed and audited_sha is untouched. Gates: tree 0 errors / 12 pre-existing reminders, wiki lint 0 errors across 332 pages, release audit CLEAN, 107 anchored links clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../components/model-executor/_index.md | 6 +- .../model-executor/rules-output-contract.md | 29 ++++++++ .../model-executor/rules-platform-backends.md | 36 ++++++++++ .../model-executor/rules-runtime-hot-paths.md | 43 ++++++++++++ .../components/model-executor/rules.md | 68 +------------------ 5 files changed, 115 insertions(+), 67 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/model-executor/rules-output-contract.md create mode 100644 knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md create mode 100644 knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b629e9596..e50a0c34b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -35,7 +35,9 @@ sources: [] |---|---| | 理解共享职责和阶段边界 | [architecture](architecture.md) | | 根据 PR 描述直达 stage config、runner preprocess、stage runtime、bridge/batch 或 loader 的规则组与第一批源码 | [rules 与代码地图](rules.md) | - | runtime info、跨 stage payload、batch 与 request RNG 合同 | [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | | loader 的 dtype/config 获取、fused shard 与多模块 checkpoint 载入 | [loader 合同](rules-loader-contract.md) | -| shared image example envelope 与 model_extras 参数声明 | [image task envelope 合同](rules-image-task-envelope.md) | \ No newline at end of file +| shared image example envelope 与 model_extras 参数声明 | [image task envelope 合同](rules-image-task-envelope.md) | +| Omni 输出类型与字段/复制合同 | [输出类型合同](rules-output-contract.md) | +| 采样循环不变量、热路径缓存、AR 音频侧路 | [运行时热路径合同](rules-runtime-hot-paths.md) | +| NPU runner、ROCm 分页注意力、NPU 模型补丁 | [平台后端合同](rules-platform-backends.md) | \ No newline at end of file diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-output-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-output-contract.md new file mode 100644 index 000000000..ac58b57da --- /dev/null +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-output-contract.md @@ -0,0 +1,29 @@ +--- +title: "Omni 输出类型合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, model-executor] +sources: ["PR #5146", "PR #6152", vllm_omni/outputs/] +confidence: high +--- + +# Omni 输出类型合同 + +`EXEC-7a`–`EXEC-7b`：Omni 输出必须是扁平的 `RequestOutput` 子类，并保持其字段与复制合同。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 + +## EXEC-7a — Omni 输出必须是扁平的 RequestOutput 子类 + +- 触发：修改 `OmniRequestOutput`、stage output wrapping、`omni.generate()` 返回对象或其 serving/offline 消费者。 +- 强制：`OmniRequestOutput` 必须继承 vLLM `RequestOutput` 并将继承字段作为 dataclass 字段直接声明；stage 原始输出必须通过 `from_stage_output()` 显式复制 `prompt`、`prompt_token_ids`、`outputs`、`finished` 等生成内容，源对象为 `OmniRequestOutput` 时再复制 diffusion 内容；消费者直接读取扁平字段，列表归一化只在真实列表边界完成。 +- 禁止：重新引入 `request_output` 嵌套字段、动态 pass-through property、修改 dataclass 生成的 `__init__` 或递归 `unwrap` 兼容层；不得在 serving、example 或工具代码中恢复 `output.request_output` 访问。 +- 验收：用真实 `RequestOutput` 和 `OmniRequestOutput` 分别验证 factory 复制的 prompt、token、outputs、finished 及 images/trajectory 内容；覆盖文本、音频、图像和 pipeline stage 的直接字段消费，并确认未完成 stage 的 `finished` 不被包装过程改写。^[PR #5146] + +## EXEC-7b — OmniRequestOutput 必须保持 RequestOutput 字段与复制合同 + +- 触发：修改 `OmniRequestOutput`、升级 vLLM `RequestOutput`，或调整 `from_stage_output()` 的输出复制逻辑时。 +- 强制：对照真实 `RequestOutput.__init__` 设置的全部公开属性，在 `OmniRequestOutput` 中以匹配的类型和默认值声明字段，并同步加入 `_REQUEST_OUTPUT_CONTENT_ATTRS`，确保 `from_stage_output()` 复制非空值。 +- 禁止：只声明新字段而遗漏复制列表，或用手写的静态字段清单代替真实 `RequestOutput` 属性 parity 检查，导致 serving 所需的 connector metadata 或 cache accounting 静默丢失。 +- 验收：用真实 `RequestOutput` 与 `OmniRequestOutput` 做属性 parity 测试；覆盖新字段默认值为 `None`，以及 `ec_transfer_params`、`num_cache_creation_tokens` 等非空值经 `from_stage_output()` 后保持不变。 ^[PR #6152] + +相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md new file mode 100644 index 000000000..1fcf7ce41 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -0,0 +1,36 @@ +--- +title: "平台后端合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, model-executor] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/] +confidence: high +--- + +# 平台后端合同 + +`EXEC-10a`、`EXEC-12a`–`EXEC-13a`：NPU 平台 runner 的 runtime mode 与 dummy-run 接口、ROCm 分页注意力的 packed KV varlen 路径，以及 NPU 模型补丁的注册与回归。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 + +## EXEC-10a — NPU 平台 runner 必须保持 runtime mode 与 dummy-run 接口合同 + +- 触发：平台 runner 覆盖 `dummy_run` 或接收 `cudagraph_runtime_mode`，而共享 runner 新增运行时参数或需要校验复合 runtime mode。 +- 强制：平台 override 必须保留共享签名中的 `randomize_inputs` 默认值并将其传给 `maybe_randomize_inputs`；runtime mode 必须调用 `is_valid_runtime_mode()` 判断当前实例是否有效。 +- 禁止：通过实例调用返回集合的 `valid_runtime_modes()` 冒充实例有效性校验，或因共享签名新增可选参数而让平台 override 产生接口漂移。 +- 验收：对 base NPU runner 与 generation override 分别覆盖 `randomize_inputs` 的默认和显式路径，并用包含 `FULL_DECODE_ONLY` 等复合模式的正负用例断言 `is_valid_runtime_mode()` 的校验结果和签名 parity。 ^[PR #6096] + +## EXEC-12a — ROCm 分页注意力必须走 packed KV 的兼容 varlen 路径 + +- 触发：修改 `vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py` 的 ROCm 分页注意力、AITER/`flash_attn` varlen kernel、`block_table` 映射或 KV block layout。 +- 强制：ROCm 路径必须避开 CUDA 专用的 `vllm.vllm_flash_attn` 接口，优先解析 AITER、再回退到 `flash_attn`；依据每个请求的 `seq_lens` 构造 `cu_seqlens_k`，在 device 上按 `block_table` gather 可见 KV 为 packed tensors，并使用匹配的 query offsets、最大长度、`softmax_scale` 和 `causal` 参数调用 `flash_attn_varlen_func`。 +- 禁止：复用 CUDA 专用的 `seqused_k`/`fa_version` 参数合同；直接把不兼容的 ROCm paged kernel 用于 AR-Diffusion 的 frame-aligned blocks；依赖 ROCm kernel 对 `block_table` 的隐式支持；丢失逐请求长度或把 KV gather 到 CPU 后再调用 kernel。 +- 验收：在 ROCm GPU 上覆盖不同 history/action 长度、block table、请求长度和 `commit_current` 组合，与 dense reference 校验数值和输出 shape；分别验证 AITER 与 `flash_attn` fallback，确认 HIP 路径不导入 `vllm.vllm_flash_attn`，并确认 CUDA/CPU 路径行为不变。 ^[PR #5886] + +## EXEC-13a — NPU 模型补丁必须由生产初始化路径注册并可回归 + +- 触发：在 NPU platform 初始化中加入模型专有 monkey patch、融合算子或其他必须全局启用的 model hook。 +- 强制：把 hook 封装成幂等的 `apply_*_patch()`，在 `adapt_patch(is_global_patch=True)` 完成后由生产初始化路径显式调用；注册必须只指向目标模型 consumer，并保留非 NPU和其他模型路径。测试必须验证生产注册链，而不只直接调用 helper。 +- 禁止：依赖 import side effect 或手工调用 helper 代替平台注册；只覆盖 patched function 的数值而不验证注册实际生效；把模型专有 patch 扩大到 MiniMax H3 DiT 或所有 NPU 模型。 +- 验收：从 NPU 平台初始化/注册入口运行测试，断言目标 Qwen3-VL text encoder 的 consumer 已替换且重复初始化不重复 patch；随后用真实 consumer 覆盖至少一个 BNSD fast path、一个 BSND fallback 和 batched M-RoPE。^[PR #6061] + +相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md new file mode 100644 index 000000000..cbd4184ea --- /dev/null +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md @@ -0,0 +1,43 @@ +--- +title: "运行时热路径合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, model-executor] +sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/] +confidence: high +--- + +# 运行时热路径合同 + +`EXEC-8a`、`EXEC-9a`、`EXEC-11a`–`EXEC-11b`：采样循环的不变量提取、固定输入缓存与掩码/精度边界，以及连续 AR 音频侧路和 codec 帧账本。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 + +## EXEC-8a — DiT Euler 采样循环必须只计算一次不变条件 + +- 触发：修改 GLM-TTS flow-matching DiT 的 Euler 采样循环、文本 embedding、RoPE、block-causal mask、CFG batch，或新增预计算输入参数。 +- 强制：在每次 `_do_sample` 调用中只计算一次不随 timestep 或采样状态变化的 text embedding、RoPE、attention mask 和 CFG 双 batch 张量，并传入 DiT；同时保持 `seq_len`、设备、dtype 与 `forward()` 的合同一致，缺省参数仍保留原有调用行为。 +- 禁止：在每个 Euler step 重复构造上述固定输入；让预计算 embedding 或 RoPE 在长度不匹配时静默继续；改变原有 mask device、模型 dtype 或 block-causal 语义；把 eager streaming 路径的优化结论外推为 CUDA-graph 路径或其他模型的通用结论。 +- 验收：覆盖 CFG/non-CFG 和 block-causal streaming 路径，验证预计算输入的长度、设备、dtype 与采样序列匹配，并以数值/字节一致性、输出 shape 及 profiling 证明循环内不再重复执行固定计算；确认非 streaming CUDA-graph 和未提供预计算参数的旧调用仍正常。 ^[PR #5068] + +## EXEC-9a — OmniVoice 热路径的固定输入缓存与掩码/精度边界 + +- 触发：修改 OmniVoice 离散扩散生成循环中的 D2H 同步、attention mask、RoPE、文本/音频 embedding、CFG、CUDA graph/eager 路径或 TF32 配置。 +- 强制：将循环不变的文本 embedding、`audio_mask_3d`、RoPE 和 additive attention mask 在循环外缓存；批量取得 `c_lens` 并只在实际消费的 logits slice 上转为 fp32；CUDA graph 与 eager 使用相同的 float mask 语义，bool mask 必须将 True 映射为 `0.0`、False 映射为 `-inf`，已有 float mask 必须原样保留；TF32 只能通过显式配置 opt-in，并在 graph capture 前启用。 +- 禁止：每步重复计算固定输入或执行逐请求 D2H `.item()`；对 float mask 使用布尔反转或把 bool 直接隐式转换进 float buffer；让 graph capture 与 replay 使用不同 mask dtype；默认开启或宣称 TF32 路径 bit-identical；只更新 conditional 或 unconditional 一侧的迭代 token。 +- 验收：覆盖 eager 与 CUDA graph、bool 与 additive float mask、CFG/non-CFG 及不同 batch/concurrency，断言 masked 位置仍为 `-inf`、float mask 数值保留、graph/eager 输出满足既定精度合同、每步无固定输入重算和逐请求同步；单独验证 `enable_tf32` 默认关闭且显式开启前后性能/非 bit-identical 语义有记录。 ^[PR #5174] + +## EXEC-11a — 连续 AR 音频侧路必须保持 fp32、上下文与正常终止 + +- 触发：vLLM-native AR 模型在每个 token/patch 旁路执行 DiT 或 flow-matching 采样、连续声码器解码，并通过 Omni multimodal output 输出增量音频。 +- 强制：Euler/ODE 累积状态保持 fp32，仅将 DiT matmul 输入置为模型 dtype；跨 patch 保留 causal decoder 的 streaming window，并在模型停止或容量边界通过正常 stop 信号前 flush lookahead；输出按 step 发 delta audio，并保留 `meta.sparse_audio` 等路由标记。 +- 禁止：以 bf16 作为多步积分默认累积 dtype；每个 patch 独立零填充解码；容量耗尽时 raise 使 engine 失效；停止时丢弃 decoder 尾部，或让 scaffold hidden 通过默认 multimodal merge 混入音频。 +- 验收：跨两次 engine restart 做 bit-identical/确定性探针，做跨 patch 首词与边界上下文验证；将容量缩小后确认请求正常截断且同一 engine 的下一次 generate 仍成功，并断言逐步 delta 合并后只有目标音频张量。^[PR #4765] + +## EXEC-11b — codec 音频流必须由逐请求帧账本驱动 + +- 触发：native-AR 音频模型逐步产生多头 codec codes，按 chunk 解码并通过 sparse multimodal output 交付增量 waveform 时。 +- 强制：按 request id 保存 frame history 与 `emitted_frames` 账本；仅提交有效音频 frame；每个 decode window 带 lookback 并在裁剪后用实际 decode 结果推导 samples-per-frame；以原 request id 排队增量 payload，在 STOP 或预算最后一步、request 仍在输出 batch 时完成最终 flush，并保留 `meta.sparse_audio`。 +- 禁止：在裸 chunk 边界解码、假定标称帧率或固定 hop、每帧解码或每次发送完整历史；把尾部音频放进 finished cleanup；让已离开输出 batch 的 request 继续产生不可路由 payload，或以 pooler/hidden state 充当音频。 +- 验收：CPU codec stand-in 覆盖跨 first/steady chunk 边界的完整帧拼接、lookback 和 partial final chunk；真实 runner 覆盖 mixed requests、sparse routing、预算尾部与 cleanup；GPU smoke 断言 waveform 样本数等于 committed frames 的实际 frame ledger。 ^[PR #5666] + +相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 4e5f3f0ab..e1996ef21 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -37,6 +37,9 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | | `image-task-envelope` | shared image task example 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6b`，见 [image task envelope 合同](rules-image-task-envelope.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | +| `output-contract` | Omni 输出类型与字段/复制合同 | `EXEC-7a`–`EXEC-7b`，见 [输出类型合同](rules-output-contract.md) | +| `runtime-hot-paths` | 采样循环不变量、固定输入缓存、AR 音频侧路与 codec 帧账本 | `EXEC-8a`, `EXEC-9a`, `EXEC-11a`–`EXEC-11b`，见 [运行时热路径合同](rules-runtime-hot-paths.md) | +| `platform-backends` | NPU runner 接口、ROCm 分页注意力、NPU 模型补丁注册 | `EXEC-10a`, `EXEC-12a`–`EXEC-13a`，见 [平台后端合同](rules-platform-backends.md) | ## 严格配置校验 @@ -152,68 +155,3 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru 当 issue 已提供最终配置和失败堆栈、current source 可读时，先按 [架构职责锚点](architecture.md#当前源码职责锚点) 完成一轮窄调查，目标是在 owner 确定后两分钟内给出首次根因。超时仍未闭环时必须指出缺失证据，不能静默扩大到历史 commit、其他模型或环境猜测。 Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置日志加全局/per-stage 合并函数；一处启动前容量校验及其完整异常控制流；一处与日志一致的 worker 失败点。三段一致即可决定“配置触发 + fail-fast 缺陷”的主要修复位置，不再为首次结论读取 config factory、模型 pipeline、完整 deploy、spawn 实现或 tag diff；只有三段之间发生冲突时才补这些文件。 - -## 跨 stage bridge 与 batch 合同 - -### EXEC-7a — Omni 输出必须是扁平的 RequestOutput 子类 - -- 触发：修改 `OmniRequestOutput`、stage output wrapping、`omni.generate()` 返回对象或其 serving/offline 消费者。 -- 强制：`OmniRequestOutput` 必须继承 vLLM `RequestOutput` 并将继承字段作为 dataclass 字段直接声明；stage 原始输出必须通过 `from_stage_output()` 显式复制 `prompt`、`prompt_token_ids`、`outputs`、`finished` 等生成内容，源对象为 `OmniRequestOutput` 时再复制 diffusion 内容；消费者直接读取扁平字段，列表归一化只在真实列表边界完成。 -- 禁止：重新引入 `request_output` 嵌套字段、动态 pass-through property、修改 dataclass 生成的 `__init__` 或递归 `unwrap` 兼容层；不得在 serving、example 或工具代码中恢复 `output.request_output` 访问。 -- 验收：用真实 `RequestOutput` 和 `OmniRequestOutput` 分别验证 factory 复制的 prompt、token、outputs、finished 及 images/trajectory 内容；覆盖文本、音频、图像和 pipeline stage 的直接字段消费，并确认未完成 stage 的 `finished` 不被包装过程改写。^[PR #5146] - -### EXEC-7b — OmniRequestOutput 必须保持 RequestOutput 字段与复制合同 - -- 触发：修改 `OmniRequestOutput`、升级 vLLM `RequestOutput`，或调整 `from_stage_output()` 的输出复制逻辑时。 -- 强制：对照真实 `RequestOutput.__init__` 设置的全部公开属性，在 `OmniRequestOutput` 中以匹配的类型和默认值声明字段，并同步加入 `_REQUEST_OUTPUT_CONTENT_ATTRS`，确保 `from_stage_output()` 复制非空值。 -- 禁止：只声明新字段而遗漏复制列表，或用手写的静态字段清单代替真实 `RequestOutput` 属性 parity 检查，导致 serving 所需的 connector metadata 或 cache accounting 静默丢失。 -- 验收：用真实 `RequestOutput` 与 `OmniRequestOutput` 做属性 parity 测试；覆盖新字段默认值为 `None`，以及 `ec_transfer_params`、`num_cache_creation_tokens` 等非空值经 `from_stage_output()` 后保持不变。 ^[PR #6152] - -### EXEC-8a — DiT Euler 采样循环必须只计算一次不变条件 - -- 触发：修改 GLM-TTS flow-matching DiT 的 Euler 采样循环、文本 embedding、RoPE、block-causal mask、CFG batch，或新增预计算输入参数。 -- 强制：在每次 `_do_sample` 调用中只计算一次不随 timestep 或采样状态变化的 text embedding、RoPE、attention mask 和 CFG 双 batch 张量，并传入 DiT；同时保持 `seq_len`、设备、dtype 与 `forward()` 的合同一致，缺省参数仍保留原有调用行为。 -- 禁止：在每个 Euler step 重复构造上述固定输入；让预计算 embedding 或 RoPE 在长度不匹配时静默继续；改变原有 mask device、模型 dtype 或 block-causal 语义；把 eager streaming 路径的优化结论外推为 CUDA-graph 路径或其他模型的通用结论。 -- 验收：覆盖 CFG/non-CFG 和 block-causal streaming 路径，验证预计算输入的长度、设备、dtype 与采样序列匹配，并以数值/字节一致性、输出 shape 及 profiling 证明循环内不再重复执行固定计算；确认非 streaming CUDA-graph 和未提供预计算参数的旧调用仍正常。 ^[PR #5068] - -### EXEC-9a — OmniVoice 热路径的固定输入缓存与掩码/精度边界 - -- 触发：修改 OmniVoice 离散扩散生成循环中的 D2H 同步、attention mask、RoPE、文本/音频 embedding、CFG、CUDA graph/eager 路径或 TF32 配置。 -- 强制：将循环不变的文本 embedding、`audio_mask_3d`、RoPE 和 additive attention mask 在循环外缓存；批量取得 `c_lens` 并只在实际消费的 logits slice 上转为 fp32；CUDA graph 与 eager 使用相同的 float mask 语义，bool mask 必须将 True 映射为 `0.0`、False 映射为 `-inf`，已有 float mask 必须原样保留；TF32 只能通过显式配置 opt-in，并在 graph capture 前启用。 -- 禁止：每步重复计算固定输入或执行逐请求 D2H `.item()`；对 float mask 使用布尔反转或把 bool 直接隐式转换进 float buffer；让 graph capture 与 replay 使用不同 mask dtype；默认开启或宣称 TF32 路径 bit-identical；只更新 conditional 或 unconditional 一侧的迭代 token。 -- 验收：覆盖 eager 与 CUDA graph、bool 与 additive float mask、CFG/non-CFG 及不同 batch/concurrency，断言 masked 位置仍为 `-inf`、float mask 数值保留、graph/eager 输出满足既定精度合同、每步无固定输入重算和逐请求同步；单独验证 `enable_tf32` 默认关闭且显式开启前后性能/非 bit-identical 语义有记录。 ^[PR #5174] - -### EXEC-10a — NPU 平台 runner 必须保持 runtime mode 与 dummy-run 接口合同 - -- 触发：平台 runner 覆盖 `dummy_run` 或接收 `cudagraph_runtime_mode`，而共享 runner 新增运行时参数或需要校验复合 runtime mode。 -- 强制：平台 override 必须保留共享签名中的 `randomize_inputs` 默认值并将其传给 `maybe_randomize_inputs`；runtime mode 必须调用 `is_valid_runtime_mode()` 判断当前实例是否有效。 -- 禁止：通过实例调用返回集合的 `valid_runtime_modes()` 冒充实例有效性校验，或因共享签名新增可选参数而让平台 override 产生接口漂移。 -- 验收：对 base NPU runner 与 generation override 分别覆盖 `randomize_inputs` 的默认和显式路径，并用包含 `FULL_DECODE_ONLY` 等复合模式的正负用例断言 `is_valid_runtime_mode()` 的校验结果和签名 parity。 ^[PR #6096] - -### EXEC-11a — 连续 AR 音频侧路必须保持 fp32、上下文与正常终止 - -- 触发：vLLM-native AR 模型在每个 token/patch 旁路执行 DiT 或 flow-matching 采样、连续声码器解码，并通过 Omni multimodal output 输出增量音频。 -- 强制：Euler/ODE 累积状态保持 fp32，仅将 DiT matmul 输入置为模型 dtype；跨 patch 保留 causal decoder 的 streaming window，并在模型停止或容量边界通过正常 stop 信号前 flush lookahead；输出按 step 发 delta audio，并保留 `meta.sparse_audio` 等路由标记。 -- 禁止：以 bf16 作为多步积分默认累积 dtype；每个 patch 独立零填充解码；容量耗尽时 raise 使 engine 失效；停止时丢弃 decoder 尾部，或让 scaffold hidden 通过默认 multimodal merge 混入音频。 -- 验收：跨两次 engine restart 做 bit-identical/确定性探针，做跨 patch 首词与边界上下文验证；将容量缩小后确认请求正常截断且同一 engine 的下一次 generate 仍成功，并断言逐步 delta 合并后只有目标音频张量。^[PR #4765] - -### EXEC-11b — codec 音频流必须由逐请求帧账本驱动 - -- 触发：native-AR 音频模型逐步产生多头 codec codes，按 chunk 解码并通过 sparse multimodal output 交付增量 waveform 时。 -- 强制：按 request id 保存 frame history 与 `emitted_frames` 账本；仅提交有效音频 frame；每个 decode window 带 lookback 并在裁剪后用实际 decode 结果推导 samples-per-frame；以原 request id 排队增量 payload，在 STOP 或预算最后一步、request 仍在输出 batch 时完成最终 flush，并保留 `meta.sparse_audio`。 -- 禁止：在裸 chunk 边界解码、假定标称帧率或固定 hop、每帧解码或每次发送完整历史；把尾部音频放进 finished cleanup；让已离开输出 batch 的 request 继续产生不可路由 payload，或以 pooler/hidden state 充当音频。 -- 验收：CPU codec stand-in 覆盖跨 first/steady chunk 边界的完整帧拼接、lookback 和 partial final chunk；真实 runner 覆盖 mixed requests、sparse routing、预算尾部与 cleanup；GPU smoke 断言 waveform 样本数等于 committed frames 的实际 frame ledger。 ^[PR #5666] - -### EXEC-12a — ROCm 分页注意力必须走 packed KV 的兼容 varlen 路径 - -- 触发：修改 `vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py` 的 ROCm 分页注意力、AITER/`flash_attn` varlen kernel、`block_table` 映射或 KV block layout。 -- 强制：ROCm 路径必须避开 CUDA 专用的 `vllm.vllm_flash_attn` 接口，优先解析 AITER、再回退到 `flash_attn`；依据每个请求的 `seq_lens` 构造 `cu_seqlens_k`，在 device 上按 `block_table` gather 可见 KV 为 packed tensors，并使用匹配的 query offsets、最大长度、`softmax_scale` 和 `causal` 参数调用 `flash_attn_varlen_func`。 -- 禁止：复用 CUDA 专用的 `seqused_k`/`fa_version` 参数合同；直接把不兼容的 ROCm paged kernel 用于 AR-Diffusion 的 frame-aligned blocks；依赖 ROCm kernel 对 `block_table` 的隐式支持；丢失逐请求长度或把 KV gather 到 CPU 后再调用 kernel。 -- 验收：在 ROCm GPU 上覆盖不同 history/action 长度、block table、请求长度和 `commit_current` 组合，与 dense reference 校验数值和输出 shape；分别验证 AITER 与 `flash_attn` fallback，确认 HIP 路径不导入 `vllm.vllm_flash_attn`，并确认 CUDA/CPU 路径行为不变。 ^[PR #5886] - -### EXEC-13a — NPU 模型补丁必须由生产初始化路径注册并可回归 - -- 触发：在 NPU platform 初始化中加入模型专有 monkey patch、融合算子或其他必须全局启用的 model hook。 -- 强制：把 hook 封装成幂等的 `apply_*_patch()`，在 `adapt_patch(is_global_patch=True)` 完成后由生产初始化路径显式调用；注册必须只指向目标模型 consumer，并保留非 NPU和其他模型路径。测试必须验证生产注册链，而不只直接调用 helper。 -- 禁止：依赖 import side effect 或手工调用 helper 代替平台注册；只覆盖 patched function 的数值而不验证注册实际生效；把模型专有 patch 扩大到 MiniMax H3 DiT 或所有 NPU 模型。 -- 验收：从 NPU 平台初始化/注册入口运行测试，断言目标 Qwen3-VL text encoder 的 consumer 已替换且重复初始化不重复 patch；随后用真实 consumer 覆盖至少一个 BNSD fast path、一个 BSND fallback 和 batched M-RoPE。^[PR #6061] From 5f32c9a8f106a553a63fa999679e73223cbb2fcf Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 05:58:14 +0800 Subject: [PATCH 177/465] knowledge(vllm-omni): sync 3875f8de (PR #5491) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 3875f8ded41d5015aa98e5c35f12e6d2770c15b1 Upstream-PR: #5491 Knowledge-Delta: 新增 AR-Diffusion tick/session 身份与 fail-closed 提交流程、跨持久状态的 paged-KV 容量预算规则，以及自定义 engine 与 worker-local lifecycle 的 serving 规则。 --- adapters/vllm_omni/release_baseline.yaml | 8 ++++---- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-system-runtime.md | 12 ++++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-bridge-batch.md | 9 ++++++++- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 12 files changed, 41 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index eac92bce4..cd470a9fd 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 68bf8c2c4296fb851c36173e5dcdca55ea56500c + previous_audited_sha: 9798bc3a8d5a400b97fba7402f42510577e7302d audited_ref: main - audited_sha: 9798bc3a8d5a400b97fba7402f42510577e7302d + audited_sha: 3875f8ded41d5015aa98e5c35f12e6d2770c15b1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 86 sha256: 039140fb0a8e95f13015d11af550db8ed16e6071448889f243c1525de5dff493 diffusion: - count: 63 - sha256: c776968386eba1c3456afa3ffe518c8e960880ed583b10e71d3f53b21ffe9cbb + count: 64 + sha256: 2d677c8989dc07d5003a429af0f56c6783c9ec8c95c32831cc80e9448918a89b pipelines: count: 57 sha256: 1cfc25b2c2bb9a543a77830477a795ec59e44d4abd3d07c9b8f6f05ec07d1aac diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 554de7ab2..9dcffe85f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -9798bc3a8d5a400b97fba7402f42510577e7302d`. The canonical machine baseline +3875f8ded41d5015aa98e5c35f12e6d2770c15b1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b1ea00384..729490e5f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9798bc3a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3875f8de` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 65a9b2a34..89178ed51 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9798bc3a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 3875f8de` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index 4af2cb70b..5efb24695 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -1,10 +1,10 @@ --- title: "Diffusion paged cache 与系统运行时规则" created: 2026-09-03 -updated: 2026-09-03 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, "PR #5491"] confidence: high --- @@ -155,3 +155,11 @@ confidence: high 相关执行流见 [Diffusion architecture](architecture.md)；paged attention 的 layout 细节见 [attention 规则](rules-attention.md)。 + +## DIFF-4o — AR-Diffusion 容量必须覆盖持久状态，分页图输入必须显式 + +- 触发：AR-Diffusion runner 修改 paged self-attention KV、resident session capacity、scratch/cross-attention reservation、model-owned CUDA state、commit pruning 或 CUDA Graph 输入。 +- 强制：按有效 resident capacity 完整计算 `sink + recent window`、单个 in-flight block、scratch、cross-attention 和每 session 的 model-owned state；`gpu_memory_fraction` 只能控制额外 residency，至少一个能装入实际空闲显存的 session 必须先被接纳，并在分配 self-KV 前扣除所有持久 reservation。成功 commit 后立即释放 skipped paged blocks；paged attention 将可变 key/value pool 作为显式可变 graph input。worker-local state 仍须保持 `max_num_seqs=1`，多副本必须先具备 session affinity。 +- 禁止：用只覆盖一个 session 的固定 floor 忽略声明 capacity；只为 self-KV 预算而漏算 cross-attention、scratch 或模型自有状态；使用 process-global KV pool registry；让各 TP rank 依据不同的本地 free-memory 计算不同 capacity 后继续 collective；把单次 TP replay 或能启动当作多 session/多 rank 预算正确性的证明。 +- 验收：覆盖 LingBot 两个 resident window、DreamZero 声明 capacity 64 的预算封顶、LRU eviction、失败 forward 清理、commit 后 block 回收和显式 graph pool 输入；真实 TP 还要用统一的 min free-memory/all-reduce 或 rank-0 广播验证所有 rank 得到相同 capacity，并分别验证 paged/direct 数值与单副本路由。^[PR #5491] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2d9e77308..ec6988fda 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9798bc3a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3875f8de` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e50a0c34b..5c41922ce 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9798bc3a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3875f8de` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md index ca5ca8360..bca50c556 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666"] +sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666", "PR #5491"] confidence: high --- @@ -125,3 +125,10 @@ Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x - 禁止：用当前 span 或模型本地计数器猜预算；只让 vLLM 外层 sampler 看到 seed；依赖 `on_requests_finished` 之后再 flush，此时 request 已离开输出 batch，payload 无法路由。 - 验收：通过真实 runner 的 mixed-batch 路径覆盖停止结束、预算结束和最后一个 in-flight request，断言每个请求的 committed frame 全部送达；覆盖同 seed、异 seed 及有无 batch 邻居的结果与 request 归属。 ^[PR #5666] +## EXEC-1l — typed AR-Diffusion tick 必须隔离协议身份并以完整 metadata 提交 + +- 触发：新增或修改实验性 AR-Diffusion tick/session、`extra_args` 传输、跨请求事件队列或 chunk 输出元数据。 +- 强制：以不可变且深拷贝隔离的 `ARDiffusionTickRequest` 作为快照；`session_id` 跨 chunk 稳定，`event_id` 在会话内单调，`chunk_index` 连续，`request_id` 只标识一个 chunk。通过 `sampling_params.extra_args["ar_diffusion_tick"]` 传输，输出在 `multimodal_output["metadata"]["ar_diffusion"]` 返回；协议 request ID 必须与 `AsyncOmni` 的内部 engine routing ID 分离。只有输出元数据与提交快照完全一致后，才能提交事件、prompt、controls 和 chunk 状态；模型失败、元数据不匹配或 reducer commit 失败都必须转为 FAILED 并关闭 worker，后续只能显式 reset 或 close。 +- 禁止：用内部 engine request ID 冒充协议 request ID；在模型输出验证前确认事件已应用；原地重试失败 chunk；让通用 session 层解析 LingBot camera/action schema；在没有 session-affine routing 时启用多副本 AR stage；把内部 tick contract 当作公共 HTTP/WebSocket 协议。 +- 验收：覆盖乱序/重复/过期事件、队列背压、交错 `A0 -> B0 -> A1`、ID 分离、缺失或错误元数据、reducer commit 失败、失败后 reset、cleanup 失败 tombstone 与 close retry；同时验证 consumer 对非单副本 stage fail closed，并断言普通 `AsyncOmni` 输出 ID 合同不变。^[PR #5491] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f5c9cbbdf..683c99dcd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9798bc3a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3875f8de` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3f734cdea..b500fc016 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9798bc3a` 验证存在 +- 源码校验：以上路径均已在 `main @ 3875f8de` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 76646b17e..fe002b6d4 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491"] confidence: high --- @@ -100,6 +100,13 @@ confidence: high - 禁止：在 diffusion 实例上无条件解析 AR-stage TTS adapter、读取仅由 `__init__` 设置的 `_adapter`，或把 pure-diffusion 请求重新路由到 AR adapter 路径；只覆盖普通构造器而不覆盖 bypass-`__init__` factory。 - 验收：用 `OmniOpenAIServingSpeech.for_diffusion()` 构造实例，提交公开 `create_speech()` 请求并断言成功的 `200`/`audio/wav` 响应；以 mock 断言 diffusion speed 请求返回原值且 `resolve_adapter()` 未被调用，同时验证普通 AR 实例仍解析并使用其 adapter，所有路径不得出现 `AttributeError`。^[PR #6121] +### SERV-5h — 自定义 AR-Diffusion engine 与 worker-local session lifecycle 必须保持一致 + +- 触发：默认 stage 构造路径接入自定义 diffusion engine backend，或新增 worker-local AR session 的 reset/close lifecycle RPC 与副本限制。 +- 强制：`load_and_resolve_stage_configs()` 在没有 deploy config 时也必须保留显式 `engine_backend`；runner 选择由选中的 engine/platform owner 负责，不重复注入 `diffusion_model_runner_cls`。session reset、close 和失败清理由选定 stage 的 collective RPC 到达实际 worker，并在所有 worker 支持后才确认成功；在 session-affine routing 尚未实现前，AR stage 必须保持单副本。 +- 禁止：丢弃 `engine_backend` 使 AR 请求静默回退普通 `DiffusionEngine`；通过默认 stage 额外注入 runner class 覆盖 engine-owned 选择；只修改客户端状态而不调用 worker lifecycle；以 request-ID 级 affinity 或 round-robin 路由承载 worker-local session；将清理失败当作已关闭并允许立即复用 session ID。 +- 验收：从真实 default-stage fallback 测试显式断言 `engine_backend` 进入最终 `engine_args`；用选定 stage ID 验证 reset/close RPC、unsupported worker 结果和失败重试；用两副本配置确认初始化前拒绝或证明持续 session affinity，并回归普通 diffusion engine 的既有 runner 选择。^[PR #5491] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5ee8f2611..a73489fdc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9798bc3a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3875f8de`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `68bf8c2c` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `9798bc3a` 本轮新增 diffusion inventory 条目 `LingBotWorldCausalDMDPipeline`，并加入实验性 AR-Diffusion tick/session 执行链；不可据此推断为稳定公开服务、任意多副本路由、超过 10 ticks、广泛硬件/量化支持或官方数值 parity。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 68bf8c2c \ - --to 9798bc3a \ + --from 9798bc3a \ + --to 3875f8de \ --repo \ --mode report-only ``` From 9839dd8372bf1228a503b35d614fdb1f9edb354f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:00:44 +0800 Subject: [PATCH 178/465] knowledge(vllm-omni): sync 9d2bb23f (PR #6213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 9d2bb23ff676746e1cea69f2366a32f925c39f50 Upstream-PR: #6213 Knowledge-Delta: 新增 MiniMax H3 loader-owned direct-mmap adapter、TP=1 DLO host-weight plan、rank-local staging 与 ordinary-loader fallback 边界。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minimax-h3/_index.md | 2 +- .../vllm-omni/models/minimax-h3/rules-deployment.md | 9 ++++++++- .../repos/vllm-omni/models/minimax-h3/rules-loading.md | 9 ++++++++- 12 files changed, 30 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cd470a9fd..d88e1c51c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9798bc3a8d5a400b97fba7402f42510577e7302d + previous_audited_sha: 3875f8ded41d5015aa98e5c35f12e6d2770c15b1 audited_ref: main - audited_sha: 3875f8ded41d5015aa98e5c35f12e6d2770c15b1 + audited_sha: 9d2bb23ff676746e1cea69f2366a32f925c39f50 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9dcffe85f..037e6baaa 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3875f8ded41d5015aa98e5c35f12e6d2770c15b1`. The canonical machine baseline +9d2bb23ff676746e1cea69f2366a32f925c39f50`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 729490e5f..ce2e03ea7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3875f8de` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9d2bb23f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 89178ed51..26699ccf8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3875f8de` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9d2bb23f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ec6988fda..52b862a4c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3875f8de` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9d2bb23f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5c41922ce..75131b9f4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3875f8de` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9d2bb23f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 683c99dcd..c5c93748f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3875f8de` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9d2bb23f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b500fc016..42d26c775 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3875f8de` 验证存在 +- 源码校验：以上路径均已在 `main @ 9d2bb23f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a73489fdc..0af7fae1a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3875f8de`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9d2bb23f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `9798bc3a` 本轮新增 diffusion inventory 条目 `LingBotWorldCausalDMDPipeline`，并加入实验性 AR-Diffusion tick/session 执行链；不可据此推断为稳定公开服务、任意多副本路由、超过 10 ticks、广泛硬件/量化支持或官方数值 parity。 +本次相对 `3875f8de` 本轮新增 MiniMax H3 的 loader-owned direct-mmap adapter、TP=1 DLO host-weight plan、rank-local staging 与 ordinary-loader fallback 规则。不得据此推断 TP>1 使用共享 mmap、任意模型具备 direct-mmap 支持，或将一次性硬件 smoke/内存数据视为生产性能保证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 9798bc3a \ - --to 3875f8de \ + --from 3875f8de \ + --to 9d2bb23f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index d4cf6245a..9015e9fc4 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 1edd404b3..99cb466ca 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910"] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213"] confidence: high --- @@ -215,3 +215,10 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule - 触发：MiniMax-H3 同时启用 online FP8 与普通 layerwise offload、distributed layerwise offload、resident layers 或 transposed Cutlass weight replay。\n- 强制：online FP8 可以与 resident/no-offload、普通 layerwise offload 以及 DLO full-weight per-rank no-AllGather 路径组合；DLO 启动必须使用 `--dlo-no-use-allgather`，保留 standard loader 生成的物理布局，并在 streaming、prefetch、resident replay 中保存和恢复真实 stride。\n- 禁止：继续把 online FP8 与所有 layerwise offload一概标记为不兼容；runtime-created FP8 进入 sharded DLO AllGather；用 `.view()` 把物理转置权重恢复成 contiguous 逻辑布局；把单次 B300 的显存或速度观察写成 H3 通用容量/性能保证。\n- 验收：H3 contract 覆盖 resident、普通 layerwise、DLO no-AllGather 与 no-offload 的启动/输出路径；online FP8 + DLO AllGather 在 loader 边界明确报错；转置权重回放逐值核对 stride，并分别记录 exact GPU、shape、steps、checkpoint 和 all-rank 资源结果。^[PR #5910] +## MMH3-3j — H3 DLO host-weight plan 必须限界在 TP=1 direct mmap + +- 触发：MiniMax H3 启用 DLO、`dlo_use_allgather`、direct checkpoint mmap、resident layers，或调整 DiT 与 text encoder 等 component 的 host storage。 +- 强制：loader 只把 dedicated DiT sources 纳入 `HostWeightPlan`，其他 text encoder 等 source 继续由 ordinary component loader 加载并参与 strict coverage；AllGather 且有效 DLO group 大于一时仅复制 rank-local persistent shard 后释放 mmap handle；rank-local/no-AllGather 保留 file-backed checkpoint views，并以两个按最大 block 限定的 pinned host staging slots 逐块打包；有效 group size 为一时即使 `dlo_use_allgather=True` 也不得执行 collective。direct mmap 只适用于 TP=1、非 HSDP、非 online quantization 的已证明布局。 +- 禁止：让 backend 自行决定 checkpoint compatibility 或混合 DiT/非 DiT source 后只跳过部分 materialization；把 TP>1 ordinary-loader fallback、HSDP、online quantization 或其他 fail-closed 路径宣称具有 node-shared mmap 节省；把 resident layers 增加描述成减少 rank-local pinned host master。 +- 验收：覆盖 dedicated/mixed source、strict non-DiT loading、AllGather shard 与 handle release、rank-local source retention、双 staging slot 上界、group size one 无 collective、TP>1 ordinary-loader fallback 及 unsupported-platform 不能静默丢弃 plan。PR 的 DP=2/TP=1 no-AllGather 与四 GPU topology smoke 只证明报告的 H3 配置和输出/资源观察；TP=2 行证明的是 ordinary-loader fallback，不是 shared-mmap memory guarantee。^[PR #6213] + diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md index d5b208291..4c68cb58e 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #5910"] +sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #5910", "PR #6213"] confidence: high --- @@ -54,3 +54,10 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection - 触发：修改 MiniMax-H3 的 `quantization="fp8"`、`diffusion-quantization-config`、Qwen3-VL text encoder linear 构造或其 checkpoint quantization safety filter。\n- 强制：H3 的 plain/global FP8 配置同时提供给 DiT 与已暴露为 `LinearBase` 的 Qwen3-VL text decoder；显式 `transformer`/`text_encoder` component map 只量化指定组件。eligible attention/MLP linears 使用 online FP8，vision tower、embedding、norm、RoPE、VAE 及模型指定 FP32 projection 保持 checkpoint precision；text encoder 仅允许 online FP8，BF16 encoder 必须剥离不兼容的 serialized ModelOpt/pre-quantized 配置。\n- 禁止：把 H3 global FP8 描述成 DiT-only，或据此自动量化任意普通 `torch.nn` encoder；把 serialized FP8/ModelOpt scale 配置直接交给没有对应 checkpoint 参数的 BF16 text encoder；为 H3 另造一套绕过 vLLM factory 的 FP8 method。\n- 验收：覆盖 global、DiT-only、text-encoder-only 与组合配置，断言各组件实际 quant method、未量化边界和 runtime prefix；serialized FP8 必须拒绝，ModelOpt config 必须从 BF16 text encoder 移除而保留 online FP8；text encoder fused shard/load contract 继续通过。^[PR #5910] +## MMH3-1l — H3 直接 mmap 必须由 loader adapter 证明布局等价 + +- 触发：修改 MiniMax H3 的直接 checkpoint mmap、grouped-QKV 布局转换、custom `weight_loader`、TP 边界或 DLO loader/backend 交接。 +- 强制：由 `DiffusersPipelineLoader` 预检完整的 DiT source coverage、runtime/checkpoint key、shape、dtype、TP topology 与 loader compatibility，并把唯一的 `HostWeightPlan` 交给 DLO；H3 grouped-QKV 的直接 mmap 变换由 loader-side adapter 声明，在 bounded block packing 时执行，普通 loader 路径仍须在 active weight loader 前完成原有 reorder/split。TP=1 才允许该 direct-mmap plan，TP>1、HSDP、online quantization、缺失/未知 custom loader 或 metadata 不匹配必须在模型 mutation 前回退 ordinary loader。 +- 禁止：用 `_supports_mmap_loading` 等 pipeline capability flag 或 parameter-attached transform 代替 adapter proof；让 backend 重扫 checkpoint、重建 runtime name 或绕过 H3 active loader；把 TP>1 ordinary-loader fallback 描述成 shared-mmap host-memory optimization。 +- 验收：adapter sentinel 验证 grouped-QKV 输出顺序且目标 parameter 不携带 mmap transform；preflight 覆盖 TP/HSDP/online quantization、缺 key、shape/dtype mismatch 与未适配 custom loader，并断言回退发生在 mutation 前；exact plan transfer 与 direct-mmap/ordinary-loader 两条路径分别完成 H3 权重完整性检查。^[PR #6213] + From 398715842d071629409d8ef1e5c154d24d74ba70 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:01:49 +0800 Subject: [PATCH 179/465] knowledge(vllm-omni): sync baba7d1e (PR #6200) Upstream-Commit: baba7d1e2ef9d326486493146b43157f7cd4f8d0 Upstream-PR: #6200 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d88e1c51c..3bae37f5a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3875f8ded41d5015aa98e5c35f12e6d2770c15b1 + previous_audited_sha: 9d2bb23ff676746e1cea69f2366a32f925c39f50 audited_ref: main - audited_sha: 9d2bb23ff676746e1cea69f2366a32f925c39f50 + audited_sha: baba7d1e2ef9d326486493146b43157f7cd4f8d0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 037e6baaa..ac972b7dc 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -9d2bb23ff676746e1cea69f2366a32f925c39f50`. The canonical machine baseline +baba7d1e2ef9d326486493146b43157f7cd4f8d0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ce2e03ea7..780750f57 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9d2bb23f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ baba7d1e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 26699ccf8..822957605 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9d2bb23f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ baba7d1e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 52b862a4c..f7d396908 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9d2bb23f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ baba7d1e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 75131b9f4..2acf9396c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9d2bb23f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ baba7d1e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c5c93748f..d62bb0a33 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9d2bb23f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ baba7d1e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 42d26c775..73cd6657b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9d2bb23f` 验证存在 +- 源码校验：以上路径均已在 `main @ baba7d1e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0af7fae1a..a50134e79 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9d2bb23f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ baba7d1e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `3875f8de` 本轮新增 MiniMax H3 的 loader-owned direct-mmap adapter、TP=1 DLO host-weight plan、rank-local staging 与 ordinary-loader fallback 规则。不得据此推断 TP>1 使用共享 mmap、任意模型具备 direct-mmap 支持，或将一次性硬件 smoke/内存数据视为生产性能保证。 +本次相对 `9d2bb23f` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 3875f8de \ - --to 9d2bb23f \ + --from 9d2bb23f \ + --to baba7d1e \ --repo \ --mode report-only ``` From 2f3b087e3525859821cf2584502fa74a2bb2bb1c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:03:53 +0800 Subject: [PATCH 180/465] knowledge(vllm-omni): sync 611c0835 (PR #6033) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 611c083587ee6506d6648b294917ec2b98d7abdd Upstream-PR: #6033 Knowledge-Delta: 沉淀 async-chunk 等待 deadline、发送失败身份追踪与 duplex prewarm request-scoped 错误处理；测试、worker 注释和 docs 路由不形成独立规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/distributed/rules.md | 9 ++++++++- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 12 files changed, 37 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3bae37f5a..0b2c20cb3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9d2bb23ff676746e1cea69f2366a32f925c39f50 + previous_audited_sha: baba7d1e2ef9d326486493146b43157f7cd4f8d0 audited_ref: main - audited_sha: baba7d1e2ef9d326486493146b43157f7cd4f8d0 + audited_sha: 611c083587ee6506d6648b294917ec2b98d7abdd # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ac972b7dc..8f88a06db 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -baba7d1e2ef9d326486493146b43157f7cd4f8d0`. The canonical machine baseline +611c083587ee6506d6648b294917ec2b98d7abdd`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 780750f57..fbbdbf769 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ baba7d1e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 611c0835` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 822957605..003fccdb1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ baba7d1e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 611c0835` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f7d396908..1c85c8560 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ baba7d1e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 611c0835` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 6b583fb2f..6e050e4f1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146"] +sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6033"] confidence: high --- @@ -49,6 +49,13 @@ confidence: high Qwen3-Omni/MiniCPM-o payload 路径各做真实拓扑 smoke。该矩阵不证明 issue #5595 中独立的 shutdown/orphan 问题已解决。 ^[PR #5744] +## DIST-1d — async-chunk 发送失败必须保留内部请求身份并可观测 + +- 触发：async-chunk sender 的队列任务被 `popleft()` 后发送抛异常，或 connector 返回失败但不抛异常。 +- 强制：从 task 内的 `request.request_id` 取得 scheduler 使用的内部请求 ID，在发送线程以锁保护的 ledger 中按请求记录首次失败原因；scheduler sweep 时原子 drain 并分别记录仍存活与已离开 scheduler 的请求。 +- 禁止：读取不存在的 `task["request_id"]`；用 `external_req_id` 代替 `self.requests` 的内部 key；发送失败后静默丢弃，或把已完成/已 abort 请求误当作仍可调度请求。 +- 验收：覆盖发送抛异常、`connector.put` 返回 `False`、内部/外部 ID 不同、同一请求多次失败、并发 record/drain，以及请求在 sweep 前后离开 scheduler；日志必须包含正确 request ID 和失败原因，且 ledger 只被消费一次。^[PR #6033] + ## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 - 触发：上游 MoE factory/oracle 或 `init_model_parallel_group` 增加 all-to-all manager 要求。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2acf9396c..cb2273586 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ baba7d1e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 611c0835` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d62bb0a33..b5095965a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ baba7d1e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 611c0835` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 06c222fc4..812c8bca4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6033"] --- # Scheduler 规则 @@ -231,6 +231,13 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 验收：分别覆盖 AR/generation scheduler 的成功、decoder 抛异常、无 decoder 和同批健康 request；断言成功 payload、`FinishReason.ERROR`、request-local 错误归属及无跨请求污染。 ^[PR #4795] +## SCHED-5f — async-chunk 等待必须有可配置的截止时间 + +- 触发：async-chunk 请求进入 `WAITING_FOR_CHUNK`，或部署配置读取 `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S`。 +- 强制：统一校验超时值，拒绝负数和非有限值，`0` 仅作为显式禁用并记录 warning；chunk adapter 在请求停等时开始计时、每次收到 chunk 后重置，并由 scheduler 及时结束超时且仍在 `self.requests` 中的请求。 +- 禁止：只保护 full-payload 的 `WAITING_FOR_INPUT`；让负数、`nan` 或 `inf` 静默关闭 deadline；按 stream 总生命周期而非 chunk 间 stall 时间计时；对已离开 scheduler 的请求调用 `finish_requests`。 +- 验收：覆盖有效值、显式零值、负数和非有限值解析，覆盖 dropped terminal chunk、producer 无响应、chunk 到达后重置、请求完成/abort 清理及同批健康请求继续运行；断言超时请求为 `FINISHED_ERROR`，并确认模块 reload 不污染后续测试。^[PR #6033] + ## SCHED-6a — async discard 的计数单位必须与 stale drain 一致 - 触发：上游 scheduler 改动异步占位、stale output 或 streaming segment replacement。 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 73cd6657b..e472703a5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ baba7d1e` 验证存在 +- 源码校验：以上路径均已在 `main @ 611c0835` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index fe002b6d4..4e19e6272 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033"] confidence: high --- @@ -155,6 +155,13 @@ confidence: high - 验收：覆盖 token/chat/embed、native/generic、三种 HTTP 输出与 WebSocket。落后后端只用显式 capability/version shim 并保留 eager fallback，不能让 CUDA 新参数成为全平台前提。^[PR #5957] +### SERV-6e — async-chunk prewarm 失败必须保持 request-scoped 并停止 duplex bookkeeping + +- 触发：async-chunk duplex 的 stage-0 首次提交需要 downstream prewarm，尤其是缺少 `prompt_token_ids` 或某个 downstream prewarm 提交失败时。 +- 强制：prewarm 失败必须发送 request-scoped、non-fatal 的 `ErrorMessage`（400、`BadRequestError`），完成 abort cleanup 并关闭 duplex session；调用方收到失败结果后立即停止，不得继续写 fence、submit timestamp 或重新注册 running counter。 +- 禁止：用 `fatal=True` 将单请求输入错误升级为 engine-wide failure；cleanup 已移除 request state 后仍返回成功或继续 duplex bookkeeping；让一个 request 的 prewarm 失败拖垮同批其他请求。 +- 验收：覆盖缺失 `prompt_token_ids`、downstream submit 失败和 duplex submit fall-through，断言错误只归属目标 request、engine/thread 仍存活、下游未被错误预热，且无 fence、时间戳、counter 或 session 残留。^[PR #6033] + ### SERV-8a — 本地 stage launch 必须临时应用 runtime.env - 触发：修改 `StageRuntime` 的本地 LLM/diffusion replica launch、`runtime_cfg.env`、stage device scope 或 launch lock。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a50134e79..8be08eb57 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ baba7d1e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 611c0835`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `9d2bb23f` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `baba7d1e` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 9d2bb23f \ - --to baba7d1e \ + --from baba7d1e \ + --to 611c0835 \ --repo \ --mode report-only ``` From deda271e44f675a1ceb36fb499e8f33c75bd01a7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:04:25 +0800 Subject: [PATCH 181/465] knowledge(vllm-omni): sync 46168231 (PR #5944) Upstream-Commit: 4616823125500375828071558f9027cdd3cd3a01 Upstream-PR: #5944 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0b2c20cb3..db60f2447 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: baba7d1e2ef9d326486493146b43157f7cd4f8d0 + previous_audited_sha: 611c083587ee6506d6648b294917ec2b98d7abdd audited_ref: main - audited_sha: 611c083587ee6506d6648b294917ec2b98d7abdd + audited_sha: 4616823125500375828071558f9027cdd3cd3a01 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8f88a06db..0349e283e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -611c083587ee6506d6648b294917ec2b98d7abdd`. The canonical machine baseline +4616823125500375828071558f9027cdd3cd3a01`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fbbdbf769..c338d643d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 611c0835` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 46168231` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 003fccdb1..238644950 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 611c0835` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 46168231` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1c85c8560..677f83302 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 611c0835` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 46168231` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cb2273586..8acd84669 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 611c0835` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 46168231` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b5095965a..425e370c0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 611c0835` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 46168231` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e472703a5..b95241d97 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 611c0835` 验证存在 +- 源码校验：以上路径均已在 `main @ 46168231` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8be08eb57..501e8bea8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 611c0835`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 46168231`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `baba7d1e` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `611c0835` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from baba7d1e \ - --to 611c0835 \ + --from 611c0835 \ + --to 46168231 \ --repo \ --mode report-only ``` From 73d0ea32056759f85a3020de8471d00035760324 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:06:19 +0800 Subject: [PATCH 182/465] knowledge(vllm-omni): sync 65b3d41b (PR #5272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2 Upstream-PR: #5272 Knowledge-Delta: 记录 TTS adapter 统一拥有模型专属采样覆盖，serving_speech 只保留通用调用链。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index db60f2447..bf79c3938 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 611c083587ee6506d6648b294917ec2b98d7abdd + previous_audited_sha: 4616823125500375828071558f9027cdd3cd3a01 audited_ref: main - audited_sha: 4616823125500375828071558f9027cdd3cd3a01 + audited_sha: 65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0349e283e..811eb03c4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4616823125500375828071558f9027cdd3cd3a01`. The canonical machine baseline +65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c338d643d..83ab9f92c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 46168231` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 65b3d41b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 238644950..22f977e80 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 46168231` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 65b3d41b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 677f83302..db0993525 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 46168231` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 65b3d41b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8acd84669..7ada05b85 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 46168231` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 65b3d41b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 425e370c0..b62147ed0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 46168231` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 65b3d41b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b95241d97..9a7ac8770 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 46168231` 验证存在 +- 源码校验：以上路径均已在 `main @ 65b3d41b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 4e19e6272..923b29576 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272"] confidence: high --- @@ -107,6 +107,13 @@ confidence: high - 禁止：丢弃 `engine_backend` 使 AR 请求静默回退普通 `DiffusionEngine`；通过默认 stage 额外注入 runner class 覆盖 engine-owned 选择；只修改客户端状态而不调用 worker lifecycle；以 request-ID 级 affinity 或 round-robin 路由承载 worker-local session；将清理失败当作已关闭并允许立即复用 session ID。 - 验收：从真实 default-stage fallback 测试显式断言 `engine_backend` 进入最终 `engine_args`；用选定 stage ID 验证 reset/close RPC、unsupported worker 结果和失败重试；用两副本配置确认初始化前拒绝或证明持续 session affinity，并回归普通 diffusion engine 的既有 runner 选择。^[PR #5491] +### SERV-5i — TTS 模型专属采样覆盖必须归 adapter 所有 + +- 触发：新增或迁移 TTS 模型专属 sampling 参数、`max_new_tokens`、动态 token 限制、stop token 或 Audex CFG/TTA 参数。 +- 强制：在已注册 adapter 的 `apply_sampling_overrides(sampling_params_list, request, prompt, request_id)` 中实现模型行为；`serving_speech` 仅负责合并 `extra_params`、调用 adapter override，再处理 `seed`。通用 `max_new_tokens` 必须通过共享 helper 深拷贝参数，模型特例保留各自的动态 token、CFG/TTA、stop-token 与 `+1` 语义；最终 detector 必须能解析到注册 adapter。 +- 禁止：把新的 model-type 分支或采样 mutation 加回 `serving_speech.py`；让 adapter 直接污染 engine 的共享默认 sampling 参数；保留没有 adapter 支撑的 legacy detector，或用只验证 dispatch 到达的测试替代最终 `SamplingParams` 结果断言。 +- 验收：覆盖 adapter registry/detection 无歧义、`extra_params -> apply_sampling_overrides -> seed` 顺序、无与有 `max_new_tokens`、CosyVoice3/GLM-TTS 动态边界、Ming stop token、Audex request-id/CFG/TTA 以及共享默认参数不变；测试必须检查传给 engine 的实际 sampling 参数。 ^[PR #5272] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 501e8bea8..a5a4f8e85 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 46168231`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 65b3d41b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `611c0835` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `46168231` 本轮记录 PR #5272：TTS 模型专属采样覆盖从 `serving_speech.py` 迁入已注册 adapter，检测结果也必须由 adapter 支撑。不得据此推断所有 TTS 支持相同采样参数，或将测试、pre-commit 预算视为 serving 行为合同。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 611c0835 \ - --to 46168231 \ + --from 46168231 \ + --to 65b3d41b \ --repo \ --mode report-only ``` From 4dad625901c63291664c8670b75bc113627a85bf Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:07:39 +0800 Subject: [PATCH 183/465] knowledge(vllm-omni): sync 309d9c3e (PR #6122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 309d9c3ea8083e73685562dbbc97c8b40ab6e0b1 Upstream-PR: #6122 Knowledge-Delta: 新增 SERV-1i，固化 image_reference 远程解码的重定向策略、错误分类与测试验收边界。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bf79c3938..53837e0f6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4616823125500375828071558f9027cdd3cd3a01 + previous_audited_sha: 65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2 audited_ref: main - audited_sha: 65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2 + audited_sha: 309d9c3ea8083e73685562dbbc97c8b40ab6e0b1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 811eb03c4..70c9809d8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2`. The canonical machine baseline +309d9c3ea8083e73685562dbbc97c8b40ab6e0b1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 83ab9f92c..f470c5f0d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 65b3d41b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 309d9c3e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 22f977e80..964cdd887 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 65b3d41b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 309d9c3e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index db0993525..f0a761fd8 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 65b3d41b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 309d9c3e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7ada05b85..21c2b7a57 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 65b3d41b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 309d9c3e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b62147ed0..18ba2d379 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 65b3d41b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 309d9c3e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9a7ac8770..860f388bc 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 65b3d41b` 验证存在 +- 源码校验：以上路径均已在 `main @ 309d9c3e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 1aafcaad6..8c79ae38b 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122"] confidence: high --- @@ -133,6 +133,13 @@ confidence: high - 禁止：以共享 `LTX2Pipeline` class 全局开启 preserve-image-size；按请求目标尺寸或路径名猜版本；跳过 revision；未经版本化回归就让 online 与 legacy offline I2V framing 政策改变。 - 验收：用 2、2.3、2.5 checkpoint metadata、Full/Distilled class、pinned revision 和 off-aspect image 覆盖 HTTP I2V，断言 legacy resize、2.5 source geometry、CRF 委托及最终尺寸各自符合合同。^[PR #6070] +## SERV-1i — image reference 重定向必须遵循媒体 URL 策略 + +- 触发：`decode_image_url` 处理 HTTP(S) `image_reference.image_url`，或修改媒体 URL 重定向策略与下载错误映射。 +- 强制：读取 vLLM 的 `envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS` 并传给 HTTPX 的 `follow_redirects`；策略关闭时在收到 3xx 后拒绝请求且不得请求重定向目标；最终非成功 HTTP 状态与网络/请求失败必须分别映射；data URL 继续走本地解码路径。 +- 禁止：硬编码或绕过媒体重定向策略；策略关闭时仍跟随可能跨主机的重定向；把 HTTP 状态错误与连接失败混为同一诊断；为 data URL 创建 HTTP client。 +- 验收：用确定性的 HTTPX transport 覆盖允许重定向并请求原路径和目标路径、禁用重定向且目标未被请求并在错误中指出 `VLLM_MEDIA_URL_ALLOW_REDIRECTS`、最终 404 与连接失败的错误区分，以及 data URL 不创建 HTTP client；再通过公开 `/v1/videos` image reference 回归验证解码图像进入 engine。该验证不覆盖未修改的 video/audio reference 或 image-edit fetcher。 ^[PR #6122] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a5a4f8e85..fc875c9fb 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 65b3d41b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 309d9c3e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `46168231` 本轮记录 PR #5272：TTS 模型专属采样覆盖从 `serving_speech.py` 迁入已注册 adapter，检测结果也必须由 adapter 支撑。不得据此推断所有 TTS 支持相同采样参数，或将测试、pre-commit 预算视为 serving 行为合同。 +本次相对 `65b3d41b` 本轮在 Serving 共享规则页新增 SERV-1i，记录 `/v1/videos` 的 `image_reference.image_url` 远程解码必须遵循 `VLLM_MEDIA_URL_ALLOW_REDIRECTS`，并区分重定向、HTTP 状态和网络错误。该规则仅覆盖 `decode_image_url`；不得据此推断 `decode_video_url`、`decode_audio_url` 或 image-edit fetcher 已获得同样修复。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 46168231 \ - --to 65b3d41b \ + --from 65b3d41b \ + --to 309d9c3e \ --repo \ --mode report-only ``` From 539a03b14ba7d19fd9377c8090e19391da228095 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:08:09 +0800 Subject: [PATCH 184/465] knowledge(vllm-omni): sync e96d3fe2 (PR #6207) Upstream-Commit: e96d3fe2a6b15c1820bf7cb79681927f5c4c899b Upstream-PR: #6207 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 53837e0f6..547734e1e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2 + previous_audited_sha: 309d9c3ea8083e73685562dbbc97c8b40ab6e0b1 audited_ref: main - audited_sha: 309d9c3ea8083e73685562dbbc97c8b40ab6e0b1 + audited_sha: e96d3fe2a6b15c1820bf7cb79681927f5c4c899b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 70c9809d8..caf7b53b4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -309d9c3ea8083e73685562dbbc97c8b40ab6e0b1`. The canonical machine baseline +e96d3fe2a6b15c1820bf7cb79681927f5c4c899b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f470c5f0d..830709cef 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 309d9c3e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e96d3fe2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 964cdd887..d598945f7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 309d9c3e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e96d3fe2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f0a761fd8..22dd80b1f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 309d9c3e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e96d3fe2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 21c2b7a57..3a464cbc6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 309d9c3e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e96d3fe2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 18ba2d379..ed2f20440 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 309d9c3e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e96d3fe2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 860f388bc..f648d16ce 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 309d9c3e` 验证存在 +- 源码校验：以上路径均已在 `main @ e96d3fe2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index fc875c9fb..24baf67a2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 309d9c3e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e96d3fe2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `65b3d41b` 本轮在 Serving 共享规则页新增 SERV-1i，记录 `/v1/videos` 的 `image_reference.image_url` 远程解码必须遵循 `VLLM_MEDIA_URL_ALLOW_REDIRECTS`，并区分重定向、HTTP 状态和网络错误。该规则仅覆盖 `decode_image_url`；不得据此推断 `decode_video_url`、`decode_audio_url` 或 image-edit fetcher 已获得同样修复。 +本次相对 `309d9c3e` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 65b3d41b \ - --to 309d9c3e \ + --from 309d9c3e \ + --to e96d3fe2 \ --repo \ --mode report-only ``` From ad1a17c1b869d30002ab638f85c9ba6432096be1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:08:43 +0800 Subject: [PATCH 185/465] knowledge(vllm-omni): sync 5664e69d (PR #5681) Upstream-Commit: 5664e69dbf05486468e84f81ff1245458f3d1847 Upstream-PR: #5681 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 547734e1e..5ebe6941a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 309d9c3ea8083e73685562dbbc97c8b40ab6e0b1 + previous_audited_sha: e96d3fe2a6b15c1820bf7cb79681927f5c4c899b audited_ref: main - audited_sha: e96d3fe2a6b15c1820bf7cb79681927f5c4c899b + audited_sha: 5664e69dbf05486468e84f81ff1245458f3d1847 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index caf7b53b4..ca6fff93e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e96d3fe2a6b15c1820bf7cb79681927f5c4c899b`. The canonical machine baseline +5664e69dbf05486468e84f81ff1245458f3d1847`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 830709cef..8da43616f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e96d3fe2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5664e69d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d598945f7..b647f2455 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e96d3fe2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5664e69d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 22dd80b1f..53a4cb977 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e96d3fe2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5664e69d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3a464cbc6..c1716a2ef 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e96d3fe2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5664e69d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ed2f20440..b2219f62a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e96d3fe2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5664e69d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f648d16ce..09e31b3c1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e96d3fe2` 验证存在 +- 源码校验：以上路径均已在 `main @ 5664e69d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 24baf67a2..6c0d84cf6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e96d3fe2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5664e69d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `309d9c3e` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `e96d3fe2` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 309d9c3e \ - --to e96d3fe2 \ + --from e96d3fe2 \ + --to 5664e69d \ --repo \ --mode report-only ``` From 31dea429ba73c50cb10bfb67942f95c3b817b24c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:16:43 +0800 Subject: [PATCH 186/465] knowledge(vllm-omni): sync 2a9f6aaa (PR #4222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 2a9f6aaa8e470eda306a7f2890c40025e69df9bc Upstream-PR: #4222 Knowledge-Delta: 新增 Pi0 模型合同，并补充其单 stage 配置闭合与非媒体 action 输出的共享审查门禁。 --- adapters/vllm_omni/release_baseline.yaml | 16 +++++++------- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/rules.md | 9 +++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-output-lifecycle.md | 9 +++++++- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 +++---- .../repos/vllm-omni/models/pi0/_index.md | 2 ++ knowledge/repos/vllm-omni/models/pi0/rules.md | 22 +++++++++++++++++++ 13 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/pi0/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5ebe6941a..76f1258a8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e96d3fe2a6b15c1820bf7cb79681927f5c4c899b + previous_audited_sha: 5664e69dbf05486468e84f81ff1245458f3d1847 audited_ref: main - audited_sha: 5664e69dbf05486468e84f81ff1245458f3d1847 + audited_sha: 2a9f6aaa8e470eda306a7f2890c40025e69df9bc # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,14 +13,14 @@ inventories: count: 86 sha256: 039140fb0a8e95f13015d11af550db8ed16e6071448889f243c1525de5dff493 diffusion: - count: 64 - sha256: 2d677c8989dc07d5003a429af0f56c6783c9ec8c95c32831cc80e9448918a89b + count: 65 + sha256: b0a814ade12af087b87d45e0a0583cf844456851b3dc052496bb996823b4bbb4 pipelines: - count: 57 - sha256: 1cfc25b2c2bb9a543a77830477a795ec59e44d4abd3d07c9b8f6f05ec07d1aac + count: 58 + sha256: dcec9759334dd9aff66464720aaded39c886dff9d84ec2e0ca6950a454db2df6 deploy_yamls: - count: 85 - sha256: 7cf1393ec0975196de259e9b5840c1f5465b8e9d76517c5bcdd2dbb8ce738afc + count: 86 + sha256: f766ce25bee9d46abccfe6a245ac51af99d7733c5be72b2d29032e0bbe86840f # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ca6fff93e..d102ef6c8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5664e69dbf05486468e84f81ff1245458f3d1847`. The canonical machine baseline +2a9f6aaa8e470eda306a7f2890c40025e69df9bc`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8da43616f..0fdf19990 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5664e69d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2a9f6aaa` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index efc577fbc..1653af9d3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222"] --- # vLLM-Omni 配置开发门禁 @@ -236,6 +236,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 触发：修改 MiniCPM-o 4.5 bundled deploy YAML 或 Code2Wav connector extra 中的 CFM CUDA Graph 开关、缓存上限或配置传递。\n- 强制：`minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml`、`minicpmo_4_5_3gpu.yaml` 和 `minicpmo_4_5_8x4090.yaml` 必须一致声明 `enable_cfm_graph` 与 `cfm_max_graphs`；当前默认值为 `true` 与 `32`，并传入 `MiniCPMO45Code2Wav` 的 `_cfm_graph_config` 和 `BatchedToken2Wav`。非 CUDA 设备仍必须回退 eager。\n- 禁止：只更新一个 deploy profile、依赖未记录的默认值，或把配置存在误认为 graph 已在非 CUDA 环境生效；不得把模型专有开关扩展成所有模型的通用配置合同。\n- 验收：解析四份 deploy 配置并断言开关和上限一致，追踪非默认值到 `_cfm_graph_config` 与 `BatchedToken2Wav`；覆盖开关关闭和非 CUDA fallback，并确认 stage topology 未发生额外变化。\n^[PR #6082] +### CONF-5f — π0 的 checkpoint、registry 与 deploy 必须闭合为单 diffusion stage + +- 触发：新增或修改 π0 的 `pipeline_registry.py`、`vllm_omni/deploy/pi0.yaml`、`OmniDiffusionConfig` 的 `type: pi0` autodetect，或 `Pi0Pipeline` 的 stage declaration。 +- 强制：让 `pipeline: pi0`、`OMNI_PIPELINES["pi0"]`、`Pi0Pipeline` 与 checkpoint `type: pi0` resolver 闭合；deploy 配置只能声明一个 diffusion stage，最终输出类型为 `action`，并让 `policy_server_config` 的 `image_resolution`、`action_horizon`、`action_dim`、camera 数量与 `model_config` 保持一致。 +- 禁止：只凭 checkpoint 路径或文件存在推断 pipeline，额外添加 model-executor 层或多 stage topology，或让 OpenPI metadata 与最终 stage/model config 的动作尺寸不一致。 +- 验收：无显式 deploy 配置的 checkpoint autodetect 能解析到 `Pi0Pipeline`；解析最终逐 stage config 断言只有一个 diffusion stage、`final_output_type=action`，并核对 `pi0.yaml` 与 websocket handshake metadata 的关键字段一致。^[PR #4222] + ### CONF-6a — MOSS-TTS-Local 部署必须采用官方采样参数 - 触发：修改 MOSS-TTS-Local 的 deploy YAML 或其默认采样参数时。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b647f2455..732b34c15 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5664e69d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 2a9f6aaa` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 30549aee1..d1e3fe3a5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983"] +sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222"] confidence: high --- @@ -102,3 +102,10 @@ confidence: high - 触发：修改 diffusion result pump、共享 `_rpc_futures`/`_output_futures`、`asyncio.wait_for()` 超时、请求 abort 或 shutdown future cleanup。\n- 强制：共享 Future 在 `done()` 检查后仍必须通过 `try_set_result`/`try_set_exception` 解析，并在 helper 内捕获 `concurrent.futures.InvalidStateError`，丢弃取消或已完成 Future 的晚到结果/异常，使 singleton pump 继续运行；仅 freshly-created 且尚未共享的本地 Future 可直接解析。\n- 禁止：把 `not fut.done()` 与 `set_result()`/`set_exception()` 当作原子操作；在共享 Future 上保留未保护的 resolve 调用；让取消竞态异常逃逸并杀死 result pump，随后仍以健康检查通过推断任务会完成。\n- 验收：用 `done()` 固定返回 false 但实际已取消的 racy Future 覆盖 `COMPUTE_DONE`、`OUTPUT_READY`、batch split 与 shutdown cleanup，断言 pump 不抛 `InvalidStateError`、取消 Future 保持取消、同批健康 Future 仍完成，并回归相邻并发测试。\n^[PR #5983] +## DIFF-1s — 非媒体 diffusion 输出必须显式闭合 output type 与 post-process + +- 触发：diffusion pipeline 产生 action 等非 image/video 输出，或修改 registry 中的 `final_output_type`、post-process hook 与 multiprocess 传递路径。 +- 强制：topology 的 `final_output_type` 必须与 pipeline 实际产物一致；非媒体输出使用稳定的 `DiffusionOutput.output` key，并注册可跨 orchestrator multiprocess 边界解析的 module-level post-process callable；没有变换时显式使用 identity hook。 +- 禁止：沿用 image 默认类型、用输出 ndarray 形状猜测语义、把 action 路由成 image/video，或把局部 closure 作为跨进程 post-process hook。 +- 验收：registry 能解析 pipeline 与 post-process callable，callable 可被目标进程边界使用，最终 engine output 保留稳定 key，并由端到端测试断言 action shape、dtype 和有限性。^[PR #4222] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 53a4cb977..58bc6e96f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5664e69d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2a9f6aaa` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c1716a2ef..4a99bcb66 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5664e69d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2a9f6aaa` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b2219f62a..65a7184f7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5664e69d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2a9f6aaa` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 09e31b3c1..92af39331 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5664e69d` 验证存在 +- 源码校验：以上路径均已在 `main @ 2a9f6aaa` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6c0d84cf6..5e181f355 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5664e69d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2a9f6aaa`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `e96d3fe2` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `5664e69d` 本轮新增 Pi0 模型专属规则，并补充配置闭合与非媒体 diffusion 输出规则。规则记录了 π0 的 OpenPI/LeRobot 合同、单 stage wiring、动作输出与验证边界；不能据此推断 π0.5、其他 VLA 模型或未覆盖硬件上的生产级支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from e96d3fe2 \ - --to 5664e69d \ + --from 5664e69d \ + --to 2a9f6aaa \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/pi0/_index.md b/knowledge/repos/vllm-omni/models/pi0/_index.md index 582c40770..b23be6c05 100644 --- a/knowledge/repos/vllm-omni/models/pi0/_index.md +++ b/knowledge/repos/vllm-omni/models/pi0/_index.md @@ -47,3 +47,5 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| | 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | + +| 遇到 Pi0 专有的观测预处理、flow-matching、attention、动作输出或 OpenPI parity 问题 | [Pi0 硬门禁规则](rules.md) | \ No newline at end of file diff --git a/knowledge/repos/vllm-omni/models/pi0/rules.md b/knowledge/repos/vllm-omni/models/pi0/rules.md new file mode 100644 index 000000000..0391c9188 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/pi0/rules.md @@ -0,0 +1,22 @@ +--- +title: "Pi0 模型硬门禁" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, diffusion, serving] +sources: ["PR #4222", "vllm_omni/diffusion/models/pi0/config.py", "vllm_omni/diffusion/models/pi0/modeling_pi0.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "vllm_omni/diffusion/models/pi0/processor_pi0.py", "vllm_omni/deploy/pi0.yaml", "tests/diffusion/models/pi0/test_pi0_units.py", "tests/diffusion/models/pi0/test_pi0_parity.py", "tests/e2e/online_serving/test_pi0_expansion.py"] +confidence: high +--- + +# Pi0 模型硬门禁 + +本页承载 Pi0 专有的 VLA 观测预处理、flow-matching kernel、attention 边界与动作输出验证合同；共享输出和配置 wiring 规则见相关页面。 + +## PI0-1a — π0 的观测、attention 与动作输出必须保持 OpenPI/LeRobot 合同 + +- 触发：修改 π0 的 `Pi0Config`、`processor_pi0.py`、`modeling_pi0.py`、`pipeline_pi0.py`，或调整其 camera order、flow-matching、attention mask 与动作输出合同。 +- 强制：按 checkpoint `input_features` 顺序构造最多 `max_cameras` 个 camera slot，缺失图像用 `-1` 填充并标记为无效，state 按 `max_state_dim` 补零或截断；保持 PaliGemma 前缀与 action-expert suffix 的 4D mask、float32 mask value 和手写 eager cross-attention 语义；默认 flow-matching 输出 `actions`，形状为 `[chunk_size, max_action_dim]`，π0 默认 `[50, 32]`，`session_id`/`reset` 保持无状态语义。 +- 禁止：按传入 dict 顺序重排 camera、丢弃缺失 camera slot、在未完成 4D mask 数值 parity 前替换为普通 attention backend，或把 π0 的连续动作误当成 text/image 输出；不得把 π0 合同外推为 π0.5 支持。 +- 验收：CPU unit tests 覆盖 config、camera padding、mask、normalization 与 loader remap；固定 noise 的 CPU/float32 LeRobot parity 保持 `max|Δ| < 1e-4`，pipeline e2e 返回有限的 `[50, 32]` actions，OpenPI websocket handshake 与动作 chunk 通过独立验证。^[PR #4222] + +相关页面：[Pi0 目录](./_index.md)；[Diffusion 输出与 multiprocess runtime 规则](../../components/diffusion/rules-output-lifecycle.md)；[配置开发门禁](../../components/configuration/rules.md)。 From 29cd30eedc6607f11747cdc9b31aa3407f3614a1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:18:07 +0800 Subject: [PATCH 187/465] knowledge(vllm-omni): sync 170d4987 (PR #6267) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 170d498771dc39a5cb90b1eacddc1337b6bad6f9 Upstream-PR: #6267 Knowledge-Delta: 新增 ROCm host 与 CPU tensor device 不一致时必须使用 F.linear 的平台运行时检查规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-platform-runtime.md | 11 +++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 22 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 76f1258a8..776ab5c3c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5664e69dbf05486468e84f81ff1245458f3d1847 + previous_audited_sha: 2a9f6aaa8e470eda306a7f2890c40025e69df9bc audited_ref: main - audited_sha: 2a9f6aaa8e470eda306a7f2890c40025e69df9bc + audited_sha: 170d498771dc39a5cb90b1eacddc1337b6bad6f9 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d102ef6c8..05429bf60 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2a9f6aaa8e470eda306a7f2890c40025e69df9bc`. The canonical machine baseline +170d498771dc39a5cb90b1eacddc1337b6bad6f9`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0fdf19990..5ec15cf9f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2a9f6aaa` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 170d4987` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 732b34c15..6c03aa477 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2a9f6aaa` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 170d4987` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md index a5eed3dc0..1a4705649 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md @@ -1,10 +1,10 @@ --- title: "Diffusion 平台运行时规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #6058", vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/platforms/musa/platform.py] +sources: ["PR #6058", vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/platforms/musa/platform.py, "PR #6267", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py"] confidence: high --- @@ -33,3 +33,10 @@ confidence: high - 验收：平台单测至少覆盖 Inductor enabled、mode `NONE`、非 Inductor backend，并由 worker 测试断言 platform default 先于 model hook。PR #6058 没有新增自动测试或 MUSA runtime 运行，当前证据只证明静态实现与现有 worker consumer 接通。^[PR #6058] + +## DIFFPLAT-2a — ROCm host 上的 CPU 输入必须绕过 ROCm GEMM dispatch + +- 触发：修改 LTX phase adapter 的 unquantized linear/GEMM dispatch，或出现 ROCm host 与 CPU tensor device 不一致的测试路径。 +- 强制：当 `current_platform.is_rocm()` 且 `input_.device.type == "cpu"` 时，必须使用 `F.linear(input_, weight, bias)`，绕过没有 CPU kernel 的 `rocm_unquantized_gemm`；其他设备路径继续遵循既有 dispatch 和 batch-invariant 逻辑。 +- 禁止：仅因 host platform 是 ROCm 就把 CPU tensor 交给 ROCm GEMM；不得把该 fallback 推广为 ROCm kernel 的通用 CPU 支持或改变其他平台的 dispatch 语义。 +- 验收：ROCm host 的 CPU 输入测试必须断言结果来自 `F.linear` 且未调用 `rocm_unquantized_gemm`；同时回归 ROCm device、CUDA/其他平台及既有 batch-invariant 路径。^[PR #6267] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 58bc6e96f..a139b00b5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2a9f6aaa` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 170d4987` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4a99bcb66..6599390b1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2a9f6aaa` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 170d4987` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 65a7184f7..4951ddb38 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2a9f6aaa` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 170d4987` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 92af39331..f71fb8f44 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2a9f6aaa` 验证存在 +- 源码校验：以上路径均已在 `main @ 170d4987` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5e181f355..a9d91960c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2a9f6aaa`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 170d4987`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `5664e69d` 本轮新增 Pi0 模型专属规则，并补充配置闭合与非媒体 diffusion 输出规则。规则记录了 π0 的 OpenPI/LeRobot 合同、单 stage wiring、动作输出与验证边界；不能据此推断 π0.5、其他 VLA 模型或未覆盖硬件上的生产级支持。 +本次相对 `2a9f6aaa` 本轮新增 ROCm host 上 CPU 输入绕过 ROCm GEMM dispatch 的 diffusion 平台运行时规则。该规则不表示所有 ROCm kernel 都支持 CPU，也不构成 LTX-2 端到端质量或性能结论。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 5664e69d \ - --to 2a9f6aaa \ + --from 2a9f6aaa \ + --to 170d4987 \ --repo \ --mode report-only ``` From 0e9b8de8736f206953b3805175f24dc43d1d84db Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:21:18 +0800 Subject: [PATCH 188/465] knowledge(vllm-omni): sync 5d09cf27 (PR #5604) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 5d09cf27a98bb104506ee842ca81e0e76e47dc92 Upstream-PR: #5604 Knowledge-Delta: 新增 NPU 精确形状图捕获与失败即停合同、MiniCPM-o Code2Wav Stage 2 图边界，以及 stage-scoped deploy 配置审查门禁。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../model-executor/rules-platform-backends.md | 10 +++++++++- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../repos/vllm-omni/models/minicpm-o-4-5/rules.md | 9 ++++++++- 12 files changed, 38 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 776ab5c3c..567b42335 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2a9f6aaa8e470eda306a7f2890c40025e69df9bc + previous_audited_sha: 170d498771dc39a5cb90b1eacddc1337b6bad6f9 audited_ref: main - audited_sha: 170d498771dc39a5cb90b1eacddc1337b6bad6f9 + audited_sha: 5d09cf27a98bb104506ee842ca81e0e76e47dc92 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 05429bf60..60ed95b92 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -170d498771dc39a5cb90b1eacddc1337b6bad6f9`. The canonical machine baseline +5d09cf27a98bb104506ee842ca81e0e76e47dc92`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5ec15cf9f..52902a9b3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 170d4987` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5d09cf27` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 1653af9d3..fe2a317b6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604"] --- # vLLM-Omni 配置开发门禁 @@ -243,6 +243,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：只凭 checkpoint 路径或文件存在推断 pipeline，额外添加 model-executor 层或多 stage topology，或让 OpenPI metadata 与最终 stage/model config 的动作尺寸不一致。 - 验收：无显式 deploy 配置的 checkpoint autodetect 能解析到 `Pi0Pipeline`；解析最终逐 stage config 断言只有一个 diffusion stage、`final_output_type=action`，并核对 `pi0.yaml` 与 websocket handshake metadata 的关键字段一致。^[PR #4222] +### CONF-5g — MiniCPM-o NPU Code2Wav 开关必须按 Stage 2 传递 + +- 触发：修改 MiniCPM-o 4.5 bundled deploy YAML，或新增/调整 `code2wav_enable_npu_graph`、`code2wav_max_npu_graphs` 这类 Ascend Code2Wav graph 配置。 +- 强制：`minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml` 和 `minicpmo_4_5_3gpu.yaml` 必须在 `platforms.npu.stages` 中按 stage 声明配置；Stage 0/1 保持 `compilation_config.cudagraph_mode: PIECEWISE`，Stage 2 只通过 `additional_config` 传递 graph 开关和上限，默认值为 `true` 与 `32`，并保持最终 runner 的 outer eager 语义。所有断言必须基于 `_apply_platform_overrides` 与 `merge_pipeline_deploy` 展开的最终 stage config。 +- 禁止：把 Code2Wav NPU 开关放入 shared connector extra、全局配置或其他 stage；只修改一个 profile；以 YAML 原文或字段存在推断配置已到达 Code2Wav consumer；用全局 eager override 破坏 Stage 0/1 的 `PIECEWISE` 配置。 +- 验收：逐一解析三个 profile，断言 connector extra 不含两个 Code2Wav key，NPU 展开结果包含 Stage 0/1 的 `PIECEWISE`、Stage 2 的 `enforce_eager: true`、`code2wav_enable_npu_graph: true` 和 `code2wav_max_npu_graphs: 32`，并覆盖开关关闭和缓存上限为 0 的路径。^[PR #5604] + ### CONF-6a — MOSS-TTS-Local 部署必须采用官方采样参数 - 触发：修改 MOSS-TTS-Local 的 deploy YAML 或其默认采样参数时。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6c03aa477..56724abcf 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 170d4987` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5d09cf27` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a139b00b5..ded8802da 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 170d4987` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5d09cf27` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6599390b1..5805b9b72 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 170d4987` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5d09cf27` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index 1fcf7ce41..fcaf4cae1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604"] confidence: high --- @@ -34,3 +34,11 @@ confidence: high - 验收：从 NPU 平台初始化/注册入口运行测试，断言目标 Qwen3-VL text encoder 的 consumer 已替换且重复初始化不重复 patch；随后用真实 consumer 覆盖至少一个 BNSD fast path、一个 BSND fallback 和 batched M-RoPE。^[PR #6061] 相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 + +## EXEC-13b — NPU 精确形状图捕获必须隔离动态边界并失败即停 + +- 触发：修改 `vllm_omni/platforms/npu/graph_tools.py` 的 NPUGraph 捕获/重放、NPU 模型补丁的 tensor-only 子图，或 Token2Wav 图执行上下文。 +- 强制：图缓存 key 必须同时包含 operation、缓存模式、所有输入的完整 shape、dtype 和 device；首次匹配调用先 eager 预热，再使用 vLLM global graph pool 捕获，后续重放前把当前值复制到静态输入，并在请求状态或 streaming cache 接管前 clone 持久化输出。Python metadata、CPU 频率张量、H2D、随机数和动态 state 必须留在 eager 边界；捕获失败后必须停止该 stage 进程。 +- 禁止：捕获整个请求编排或包含动态 Python/RNG 状态的路径；使用无界 shape cache、绕过 global graph pool、在 nested capture 中重放、复用会被下一次重放覆盖的输出，或吞掉 capture、patch setup、SDPA context 的异常后继续运行。 +- 验收：CPU/mock 测试覆盖 exact-signature dispatch、cached/uncached bucket、global graph pool、persistent-output ownership、capture failure fail-stop 和异常传播；A3 硬件测试分别比较 eager 与 uncached/cached NPUGraph 的输出、shape 和 cache，并核对图数量上限与 runtime preconditions。^[PR #5604] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4951ddb38..092e15018 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 170d4987` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5d09cf27` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f71fb8f44..fc6e2efe1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 170d4987` 验证存在 +- 源码校验：以上路径均已在 `main @ 5d09cf27` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a9d91960c..a21e89dce 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 170d4987`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5d09cf27`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `2a9f6aaa` 本轮新增 ROCm host 上 CPU 输入绕过 ROCm GEMM dispatch 的 diffusion 平台运行时规则。该规则不表示所有 ROCm kernel 都支持 CPU，也不构成 LTX-2 端到端质量或性能结论。 +本次相对 `170d4987` 本轮沉淀了 PR #5604 的 NPU 精确形状 NPUGraph 通用合同、MiniCPM-o 4.5 Code2Wav 的 Stage 2 图边界，以及三个 bundled deploy profile 的 stage-scoped 配置门禁。不得据此推断整个 Stage 2、非 NPU 平台或其他模型都可进行同样的图捕获，也不得把约 15% TTFT 描述视为已具备可复现的性能回归门禁。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 2a9f6aaa \ - --to 170d4987 \ + --from 170d4987 \ + --to 5d09cf27 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 17880b580..ea42fc6cc 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604"] confidence: high --- @@ -93,6 +93,13 @@ confidence: high - 触发：修改 MiniCPM-o 4.5 Code2Wav CFM DiT estimator 的 CUDA Graph 开关、`blocks_forward_chunk` graph target、按 shape 捕获/重放、cache eviction 或 eager fallback。\n- 强制：仅在 CUDA 且显式启用时捕获 `estimator.blocks_forward_chunk`；`t_embedder`、`torch.cat`、`expand`、buffer 准备和 10-step Euler loop 保持 eager。graph key 必须包含所有输入的 shape、dtype 和 device，并使用有界 `cfm_max_graphs` LRU；`cnn_cache=None` 与 `att_cache=None` 转为 graph 可接受的零值时，必须保持与 eager `[None] * depth` 的首 chunk 数值语义，重放结果和 cache 输出需 clone。\n- 禁止：捕获整个 `_estimator_step` 或手写替代 graph target；使用无界 shape cache；在 nested capture 中重放；捕获失败后伪装成功；把 CFM graph 的配置、性能或 parity 结论外推到 HiFT、encoder、TRT 或其他模型。\n- 验收：CUDA 测试覆盖 uncached/cached shape 的 graph/eager 数值 parity、`None` cache parity、lazy capture、capture failure eager fallback 和 LRU eviction；同时覆盖 active capture/non-CUDA fallback，并确认 deploy 配置值实际到达 estimator。\n^[PR #6082] +## MCPMO-1f — Ascend Code2Wav NPUGraph 只捕获精确形状的 CFM DiT + +- 触发：MiniCPM-o 4.5 在 Ascend 上修改 Code2Wav、CFM DiT estimator、NPUGraph 开关、图缓存或 bundled deploy profile。 +- 强制：Stage 2 保持 outer `enforce_eager: true`，只将确定性的 `blocks_forward_chunk` 放入按 exact shape 捕获的 inner NPUGraph；`t_embedder`、flow encoder、HiFT/RNG、请求解析和 request-owned state commit 必须 eager。cached 与 uncached attention/CNN state 使用独立 graph bucket，`code2wav_max_npu_graphs` 默认 32，超过上限的新 shape 回 eager；`code2wav_enable_npu_graph: false` 必须能关闭该 inner graph。Token2Wav 加载前设置 `torch.npu.config.allow_internal_format = False`、`torch.npu.set_compile_mode(jit_compile=False)`，拒绝 `ASCEND_LAUNCH_BLOCKING=1`，并在 graph mode 要求 MATH SDPA。 +- 禁止：捕获整个 Code2Wav Stage 2 或依赖全局 `--enforce-eager` 代替 stage-scoped 配置；把 graph 开关放进 shared connector extra；让失败捕获继续复用可能已损坏的 allocator/RNG state；把 NPU graph 的 parity、缓存或性能结论外推到 HiFT、TRT、CUDA 或其他模型。 +- 验收：解析 `minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml` 和 `minicpmo_4_5_3gpu.yaml` 的 NPU 最终 stage config，断言 Stage 0/1 的 `PIECEWISE`、Stage 2 的 outer eager 与 additional config；CPU 测试覆盖 timestep、exact dispatch、cache bucket、clone ownership、runtime precondition、MATH 选择和 fail-stop；A3 测试覆盖 uncached/cached eager parity。任何性能声明还必须记录 exact command、A3 硬件/驱动/torch-npu、warmup、测量轮数、并发、延迟/RTF 和峰值 NPU memory。^[PR #5604] + ## MCPMO-2a — registry 使用 4.5 config/version predicate - 触发：pipeline auto-detection 看到通用 `MiniCPMO` architecture。 From a5652a396d50f839fc2cbc5027c3c15601d613e7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:21:50 +0800 Subject: [PATCH 189/465] knowledge(vllm-omni): sync 265d80b4 (PR #6115) Upstream-Commit: 265d80b4dd87f86f7cf5cbde07d248855383ce90 Upstream-PR: #6115 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 567b42335..dba33db8c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 170d498771dc39a5cb90b1eacddc1337b6bad6f9 + previous_audited_sha: 5d09cf27a98bb104506ee842ca81e0e76e47dc92 audited_ref: main - audited_sha: 5d09cf27a98bb104506ee842ca81e0e76e47dc92 + audited_sha: 265d80b4dd87f86f7cf5cbde07d248855383ce90 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 60ed95b92..60c05b7bf 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5d09cf27a98bb104506ee842ca81e0e76e47dc92`. The canonical machine baseline +265d80b4dd87f86f7cf5cbde07d248855383ce90`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 52902a9b3..fc80dd63b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5d09cf27` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 265d80b4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 56724abcf..30796bc5d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5d09cf27` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 265d80b4` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ded8802da..ec5a7e0ff 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5d09cf27` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 265d80b4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5805b9b72..8033b82fb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5d09cf27` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 265d80b4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 092e15018..1668a11b2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5d09cf27` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 265d80b4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fc6e2efe1..dd4dcf42b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5d09cf27` 验证存在 +- 源码校验：以上路径均已在 `main @ 265d80b4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a21e89dce..dfa48195c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5d09cf27`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 265d80b4`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `170d4987` 本轮沉淀了 PR #5604 的 NPU 精确形状 NPUGraph 通用合同、MiniCPM-o 4.5 Code2Wav 的 Stage 2 图边界，以及三个 bundled deploy profile 的 stage-scoped 配置门禁。不得据此推断整个 Stage 2、非 NPU 平台或其他模型都可进行同样的图捕获，也不得把约 15% TTFT 描述视为已具备可复现的性能回归门禁。 +本次相对 `5d09cf27` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 170d4987 \ - --to 5d09cf27 \ + --from 5d09cf27 \ + --to 265d80b4 \ --repo \ --mode report-only ``` From 6541a3ce67db4576ac9bf91fac2e0c9418dd3ac5 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:22:23 +0800 Subject: [PATCH 190/465] knowledge(vllm-omni): sync 2fdbf223 (PR #6275) Upstream-Commit: 2fdbf2234aeb76715618a5b236f0016f115e3e64 Upstream-PR: #6275 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index dba33db8c..76bc33bc7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5d09cf27a98bb104506ee842ca81e0e76e47dc92 + previous_audited_sha: 265d80b4dd87f86f7cf5cbde07d248855383ce90 audited_ref: main - audited_sha: 265d80b4dd87f86f7cf5cbde07d248855383ce90 + audited_sha: 2fdbf2234aeb76715618a5b236f0016f115e3e64 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 60c05b7bf..c2c92264e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -265d80b4dd87f86f7cf5cbde07d248855383ce90`. The canonical machine baseline +2fdbf2234aeb76715618a5b236f0016f115e3e64`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fc80dd63b..ce3af8e69 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 265d80b4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2fdbf223` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 30796bc5d..e22396a3f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 265d80b4` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 2fdbf223` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ec5a7e0ff..ef1fd2dd2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 265d80b4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2fdbf223` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8033b82fb..def564a8f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 265d80b4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2fdbf223` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1668a11b2..e89d7ed56 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 265d80b4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2fdbf223` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index dd4dcf42b..4ecbdf517 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 265d80b4` 验证存在 +- 源码校验：以上路径均已在 `main @ 2fdbf223` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index dfa48195c..dd47ad986 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 265d80b4`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2fdbf223`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `5d09cf27` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `265d80b4` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 5d09cf27 \ - --to 265d80b4 \ + --from 265d80b4 \ + --to 2fdbf223 \ --repo \ --mode report-only ``` From 3143ed6732e2eebc2fcbfd079ad41cfb1bf754cf Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:23:47 +0800 Subject: [PATCH 191/465] knowledge(vllm-omni): sync d7216ef5 (PR #4499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: d7216ef5144866c2f039947e1cc396b1984fcf09 Upstream-PR: #4499 Knowledge-Delta: 新增 SERV-1j，明确非流式 speech raw-audio 响应的五个 usage headers、共享统计来源及 streaming/diffusion 边界。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 76bc33bc7..fba805b6c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 265d80b4dd87f86f7cf5cbde07d248855383ce90 + previous_audited_sha: 2fdbf2234aeb76715618a5b236f0016f115e3e64 audited_ref: main - audited_sha: 2fdbf2234aeb76715618a5b236f0016f115e3e64 + audited_sha: d7216ef5144866c2f039947e1cc396b1984fcf09 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c2c92264e..a21d1f604 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2fdbf2234aeb76715618a5b236f0016f115e3e64`. The canonical machine baseline +d7216ef5144866c2f039947e1cc396b1984fcf09`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ce3af8e69..38e1f0ce1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2fdbf223` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d7216ef5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e22396a3f..30c437d82 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2fdbf223` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d7216ef5` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ef1fd2dd2..843424db5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2fdbf223` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d7216ef5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index def564a8f..fa85cee84 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2fdbf223` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d7216ef5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e89d7ed56..a2839bf6d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2fdbf223` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d7216ef5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4ecbdf517..58925874f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2fdbf223` 验证存在 +- 源码校验：以上路径均已在 `main @ d7216ef5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 8c79ae38b..498846ec3 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499"] confidence: high --- @@ -140,6 +140,13 @@ confidence: high - 禁止：硬编码或绕过媒体重定向策略；策略关闭时仍跟随可能跨主机的重定向；把 HTTP 状态错误与连接失败混为同一诊断；为 data URL 创建 HTTP client。 - 验收：用确定性的 HTTPX transport 覆盖允许重定向并请求原路径和目标路径、禁用重定向且目标未被请求并在错误中指出 `VLLM_MEDIA_URL_ALLOW_REDIRECTS`、最终 404 与连接失败的错误区分，以及 data URL 不创建 HTTP client；再通过公开 `/v1/videos` image reference 回归验证解码图像进入 engine。该验证不覆盖未修改的 video/audio reference 或 image-edit fetcher。 ^[PR #6122] +## SERV-1j — 非流式 speech raw-audio 响应通过 headers 暴露 usage + +- 触发：修改 `/v1/audio/speech` 非流式 raw-audio 响应、speech usage 计算或 usage headers，或需要对齐 streaming/batch 的 token 统计语义。 +- 强制：非流式 raw-audio 响应必须复用 `build_speech_usage` 生成的 `SpeechTokenUsage`，在 usage 可用且服务非 diffusion mode 时发送 `X-VLLM-OMNI-INPUT-TOKENS`、`X-VLLM-OMNI-OUTPUT-TOKENS`、`X-VLLM-OMNI-TOTAL-TOKENS`、`X-VLLM-OMNI-INPUT-TEXT-TOKENS` 和 `X-VLLM-OMNI-INPUT-AUDIO-TOKENS`；input 使用 text 与 ICL audio tokens，output 使用生成的 codec/audio tokens，raw audio body 保持不变。 +- 禁止：把 usage JSON 塞入 raw-audio body；用原始 stage prefill 长度替代共享 usage 语义；在 streaming headers 已发送后补发尚未确定的最终 totals；宣称 diffusion-mode 路径必然提供这些 headers。 +- 验收：测试非流式请求返回 `200`、raw audio body 未改变且五个 headers 的值与 `SpeechTokenUsage` 精确一致；覆盖 usage 不可用时不生成 headers，并回归 streaming 与 batch 的 usage 语义及 diffusion-mode 的无 headers 行为。^[PR #4499] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index dd47ad986..5ad82cc6e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2fdbf223`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d7216ef5`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `265d80b4` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `2fdbf223` 本轮记录 PR #4499 为非流式 `/v1/audio/speech` 原始音频响应增加 token usage headers 的 serving 规则；文档更新和测试变更不纳入组件知识。不能据此推断 streaming raw-audio 响应支持同样 headers，或 diffusion-mode speech server 会发出这些 headers。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 265d80b4 \ - --to 2fdbf223 \ + --from 2fdbf223 \ + --to d7216ef5 \ --repo \ --mode report-only ``` From 6b281e68290fc16497b432b6761e2e28b0b56a00 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:30:55 +0800 Subject: [PATCH 192/465] knowledge(vllm-omni): split the checkpoint and loading series out of diffusion rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit diffusion/rules.md was at 281 bytes of headroom with 91 remaining units routing to it, so the next diffusion rule would have halted a unit mid-flight. split_risk.py had been naming it for two batches; this lands the split before that happens rather than after, using the driver's own PAUSE to stop cleanly at a unit boundary instead of racing it. The DIFF-2 series is one subject -- checkpoint remap, HSDP/FSDP sharding, component quantization, artifact identity, distributed layerwise offload, FP8 linear methods, LTX checkpoint profiles, online quantization and weight layout -- but it had grown across two sections, five rules under Checkpoint 与分布式加载 and three more in the 质量阈值与资源辅助 catch-all. Extracted whole: DIFF-2a-2e, 2p, 2q, 2s -> rules-checkpoint-loading.md (12058 B) Rule IDs unchanged, the checkpoint-distributed group row points at the new page, and it is registered once in _index.md. Headroom after: rules.md 12260 B. Three inbound links broke, all of the anchorless kind that no validator can see: the text names a DIFF-2 rule while the target is the page that rule just left, so the file resolves and the reader lands somewhere the rule is not. From host-weight-runtime/_index.md and two in docs/design-doc-map.md. The campaign anchor check caught them immediately, which is the first split where they were found by a check rather than by a reviewer. Two things the extraction got wrong on the first pass. The 质量阈值与资源辅助 heading sat between DIFF-2e and DIFF-2p, so the span carried it into the new page, leaving DIFF-3a nested under Checkpoint 与分布式加载 and an empty heading above unrelated rules. And the group row claimed the whole DIFF-2a-2s range for the new page, while 2f-2j and 2r live in component-lifecycle, 2k in output-lifecycle and 2l-2o in LoRA -- the primary routing table would have sent reviewers to the wrong document for nine rules. The row now names all four owning pages. No unit is consumed and audited_sha is untouched. Gates: tree 0 errors / 13 reminders (one more page now sits near the split line and is reported by split_risk.py), wiki lint 0 errors across 334 pages, release audit CLEAN, 107 anchored links clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-checkpoint-loading.md | 134 ++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 121 +--------------- .../components/host-weight-runtime/_index.md | 4 +- .../repos/vllm-omni/docs/design-doc-map.md | 6 +- 5 files changed, 142 insertions(+), 126 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 30c437d82..a6c6f0391 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -1,7 +1,7 @@ --- title: "Diffusion" created: 2026-07-10 -updated: 2026-09-03 +updated: 2026-09-04 type: index tags: [vllm-omni, components, diffusion] sources: [] @@ -47,3 +47,4 @@ sources: [] | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | | 实验性 world-model session 生命周期、LRU 与内存统计边界 | [session state](session-state.md) | +| checkpoint remap、HSDP/FSDP、component quantization 与在线量化加载 | [checkpoint 与加载合同](rules-checkpoint-loading.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md new file mode 100644 index 000000000..c38fcf67e --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -0,0 +1,134 @@ +--- +title: "Checkpoint、加载与量化合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6279", "PR #6445", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/] +confidence: high +--- + +# Checkpoint、加载与量化合同 + +`checkpoint-distributed` 审查组的 `DIFF-2a`–`DIFF-2s`：checkpoint remap、HSDP/FSDP 分片、component quantization、artifact identity、distributed layerwise offload，以及在线量化加载与权重布局。触发条件与其余审查组见 [Diffusion 共享规则](rules.md) 的 Direct 代码快速入口。 + +## DIFF-2a — checkpoint remap 必须追到已注册且真实消费的目标 + +- 触发：增加或修改 weight mapper、scale 名称、quantization adapter 或 key resolution。 +- 强制：从序列化 key 追到目标 layer 注册的 parameter/buffer 和 forward consumer； + 多条 resolution path 必须返回对称、由合同解释的目标名。模型专属 grouped-QKV reorder、 + fused gate/up split 等布局变换必须在调用 active weight loader 前完成，让 quantization wrapper + 与 TP shard 接收最终语义布局。 +- 禁止：把 producer 有而当前 consumer 不支持的 tensor 静默过滤；必须 fold/map 或 + fail fast，并在错误中标明依赖的 upstream 版本边界。 +- 验收：测试覆盖已消费 key、未知 key、当前版本不支持的 scale 以及两条 resolution + path 的输出名；fused mixed-FP8 还须覆盖 weight/scale/shard、gate/up 顺序、nested serialized + FP8 和 combined projection，缺 shard 不得默认 0。 ^[PR #5087] ^[PR #5737] ^[PR #5848] +- 强制：读取 transformer-local `config.json` 时，disk method 与 active method 归一化后不一致 + 必须失败；disk 声明 serialized、`data_type=mx_fp` 或不同 `ignored_layers` 时应按该 component + 重建 config，不能用 online/default 配置解释 serialized 权重。目标 pin 的 + `resolve_quant_config_from_disk` 只有 Wan2.2 一个 live caller，且无 helper 直测；在补齐多模型 + caller 与 mismatch/serialized/ignored-layers 测试前，这只是局部 loading guard，不是全局自动 + checkpoint reconciliation 支持。^[PR #5839] + +## DIFF-2b — HSDP/FSDP 修复必须执行真实 fully_shard + +- 触发：改动 diffusion HSDP/FSDP 参数过滤、packed/scalar parameter 或 DeviceMesh。 +- 强制：至少用单 rank Gloo + CPU DeviceMesh 执行一次真实 `fully_shard`。 +- 禁止：只断言传给 mock 的 kwargs 后声称分布式语义已覆盖。 +- 验收：普通 float parameter 变为 DTensor；packed uint8/scalar parameter 保持本地 + identity，并覆盖 loader 的真实调用边界；replicate size 非正值拒绝，1 用 1D、>1 用 2D + DeviceMesh，测试 world size 与参数一致。 ^[PR #5088] ^[PR #5872] + +## DIFF-2c — component quantization 独立解析且 namespace 端到端一致 + +- 触发：diffusion pipeline 为 text encoder、transformer、VAE 等组件增加独立量化配置。 +- 强制：每个组件独立解析量化配置，并选择一个端到端一致的 namespace：要么 selector + 保留 `text_encoder` 等完整 component owner prefix 到持权 layer，要么 pipeline 先 resolve + 单一 component，再让内部 layer 使用 component-relative prefix。只量化明确支持的 linear， + embedding、LM head、patch/final projection 等排除项必须显式保持未量化。 +- 强制：`ComponentQuantizationConfig.resolve()` 只对 `get_quant_method` 收到的 runtime prefix 做 + longest-prefix match，未命中才落到 default；它不会自动感知 checkpoint `WeightsMapper`。 + 因此 mapper、pipeline 与 layer 必须显式维持同一 namespace，并测试重叠 prefix 的最长匹配。 +- 禁止：半途裁掉 owner prefix；resolve component 后又要求 ignored layer 带 owner；用一个 + 组件配置隐式覆盖其他组件；因为同属一个模型就量化全部 linear。 +- 验收：至少覆盖“只量化一个组件、其他组件保持 BF16”的真实构造与加载，逐层断言 + 命中/排除集合，并验证 meta-device parameter 不会被提前 move。FLUX.2 的具体边界见 + [FLUX.2 规则](../../models/flux2/rules.md)，component-relative 变体见 + [MiniMax H3 规则](../../models/minimax-h3/rules.md)；新增 quantization×offload 组合需给兼容矩阵， + 未验证 quantizer fail-closed，并核对 dtype/shape/stride。 ^[PR #5136] ^[PR #5737] + ^[PR #6279] + +## DIFF-2d — Artifact identity 只信任已验证的不可变来源 + +- 触发：新增或修改 host-weight artifact、warm restore、source identity、layout/dtype policy。 +- 强制：共享 identity、restore 和 model ABI 保持 representation-neutral，dtype/layout 只属于 + 具体 policy/producer；只有同时验证 source kind、snapshot revision、repo topology 和 blob + 位置的内容寻址仓库，才可用 blob 名代替内容哈希，其他本地文件和 symlink 必须按内容定身份。 +- 禁止：因 symlink 目标名看似 40/64 位十六进制就假设不可变；把 `bf16`、具体 layout 或首个 + producer 名写进共享 identity/restorer API；用进程内 inode/mtime guard 代替跨启动身份校验。 +- 验收：第二种 synthetic representation 复用共享 identity/restorer；任意本地十六进制命名 + symlink 在同大小内容被替换后产生不同 identity；合法 Hub snapshot→blob 拓扑仍走受控快捷路径。 + ^[PR #6445] + +## DIFF-2e — distributed layerwise offload 只能选择一个一致的权重分片合同 + +- 触发：新增 `enable_distributed_layerwise_offload`、`dlo_use_allgather`、模型 + `OffloadPlan` 或 block discovery。 +- 强制：明确 rank 本地 CPU shard、固定双 GPU buffer、H2D/AllGather stream 和模型 + block list 的 owner；`dlo_use_allgather=false` 保留 standard loader 已生成的每-rank + ready-to-run layout（没有其他分片维度时才是 full weight，TP 下则是 TP-local shard），开启 + AllGather 时不得再叠加已 DTensor-sharded 的 HSDP 参数。共享 output buffer 可按所有 hook + 的最大 block 分配，但每个 hook 调 collective 前必须按 dtype 截成自己的 + `_ag_output_sizes[dtype] == dp_size * local_shard.numel()` 前缀；block-flat-relative repoint + offset 只能在这段实际 block buffer 内解释。 +- 强制：AllGather 权重组复用既有 topology：DP>1 优先 DP group；DP=1 且 SP>1 才用 SP group。 + TP 不作为 DLO AllGather group；进入 mmap path 时 TP>1 必须因绕过 TP-aware weight-loader + callback 而拒绝。no-AllGather 只关闭 DLO 新增的权重 collective，不关闭 standard loader + 已建立的 TP/HSDP/SP 语义，也不提供跨 rank host-weight 节省。DP=SP=1 不能从“无 process + group”推断为可用的 rank-local AllGather 模式：目标 pin 的 loader 在 DLO+AllGather、支持 mmap + 时会跳过 `load_weights()`，但 backend 只有 `dp_size>1` 才执行 mmap load，两个 gate 不一致； + 修复并回归前单 rank 应使用 no-AllGather 或普通 layerwise offload。 +- 强制：非 DiT component 的 staging 必须由 `OffloadPlan` 显式声明：encoder block group + rank-local streaming 不借用 DiT AllGather group，on-demand component 由 pipeline 在真实 + encode/decode stage 成对 load/offload，resident layer 只对声明的 DiT path 生效。no-AllGather + 路径保留 standard loader 产生的 TP-local layout；若模型 loader 还做 QKV/MLP transform， + mmap 必须显式 opt out，直到 transform-before-shard 等价性有测试。 +- 禁止：用 heuristic 找不到 block 时静默假装 offload 已启用；对 HSDP 参数二次分片； + 把 CPU 内存、AllGather 同步和设备显存开销隐藏在一个泛化的 `enable_cpu_offload` 开关； + 把 max-sized shared buffer 原样交给较小 block 的 `all_gather_into_tensor`。 +- 验收：CPU/Gloo 单 rank 覆盖 DTensor wrapper、shard padding、双 buffer 和 disable + cleanup；配置测试覆盖 AllGather+HSDP 的明确失败及 no-AllGather 的允许路径。另以至少两个 + 不同 block size 验证 small/large hook 都满足 collective size equality、重建权重和 repoint。 + 当前回归用 mocked collective 检查 8/32-element、dp=2 合同；没有真实 multi-rank collective + 覆盖，PR 的 8×NPU H3 E2E 说明也缺少已填写的 commit 证据，不能升级为验证结论。 ^[PR #5802] + DLO DP collective-wave admission 见 [DIFF-2k](rules-output-lifecycle.md)；no-AllGather 的 + TP-local request 不得误走 fused batch。文档 compatibility matrix 中 + “accepted” 默认只代表配置/源码 guard；PR #5836 的 predecessor partition-path commit 为 H3 + USP4+HSDP4+no-AllGather+8 resident layers 提供一次 4×H100、3/3 completion/perf run，故该 exact + 组合不再是纯 config evidence，但没有 ordinary-offload 数值/输出质量对照，且 profiler 开启、 + final repo-root modular route 未复测，仍不得升级为 production support。TP+no-AllGather、一般 + DP+SP 等其他组合仍缺完整 model×hardware E2E。^[PR #5764] ^[PR #5839] ^[PR #5836] + +多 DiT component discovery 与 dotted-path lifecycle 合同见 +[DIFF-2f–2j 专页](rules-component-lifecycle.md)。 + +## DIFF-2p — HSDP/FSDP2 必须同时识别 legacy 与 online FP8 linear method + +- 触发：修改 diffusion HSDP/FSDP2 参数准备、online FP8 quantization method 或非连续权重布局处理。 +- 强制：同时识别 `Fp8LinearMethod` 与 `Fp8PerTensorOnlineLinearMethod`；当 FP8 权重是转置形成的非连续 view 时，先将底层 `(out, in)` row-major storage 替换为 contiguous weight 供 FSDP2 接受，再让 kernel 的 `_get_layer_params` 返回对应转置 view，且同一 kernel 只 patch 一次。 +- 禁止：只检查 legacy `Fp8LinearMethod`、把非连续 transpose view 直接交给 `fully_shard`，或因新版 online method 已被识别就宣称所有 FP8 method 和布局均兼容。 +- 验收：legacy 与 `Fp8PerTensorOnlineLinearMethod` 都覆盖 regression test，断言重写层数、weight contiguous、`(out, in)` shape，以及 kernel 返回的转置 view shape/stride；另按 HSDP/FSDP 规则执行真实 `fully_shard` 验证。^[PR #5677] + +## DIFF-2q — LTX checkpoint profile 必须贯穿 revision 与 transformer 子目录 + +- 触发：新增 LTX checkpoint/version profile、Hub `revision`、Full/Distilled transformer 配置或组件加载路径。 +- 强制：显式 `model_version` 优先于结构启发式；`revision` 必须贯穿 metadata detection、prefetch、tokenizer、全部组件、transformer config/weight source、scheduler 和 post-process sample-rate；LTX-2.5 Full/SFT 使用 `transformer_full`，distilled 与旧版本使用 `transformer`，并在缺少 Gemma4 时提示 `transformers>=5.10.1,<5.15`。 +- 禁止：按路径名识别版本；只给 config lookup 传 revision 而让权重加载回到 HEAD；用 `transformer/config.json` 解释 Full checkpoint；把 transient metadata/Hub 异常静默降级为 LTX-2 并宣称 profile 已确定。 +- 验收：固定 revision 的 converted/official split 链路逐项断言所有 loader 调用和输出 metadata；用两个 transformer config marker 验证 Full 子目录选择，覆盖 local/HF、legacy/distilled/Full profile 及缺失 encoder 版本错误。^[PR #6070] + +## DIFF-2s — 在线量化加载与 layerwise offload 必须保留物理权重布局 + +- 触发：修改在线/runtime quantization 与 layerwise offload、DLO 权重加载、非连续 FP8 权重 flatten/repoint 或加载后处理流程。\n- 强制：在线量化层必须在消费完对应 checkpoint weight 后才可流式移回 CPU，并跳过已完成层的重复 `process_weights_after_loading`；继续加载其他模型权重前释放可复用的量化缓存。对非连续权重按物理 storage 顺序打包，记录 `shape`、`stride` 和实际 storage span，并在普通 layerwise、DLO prefetch 与 resident replay 中用 `torch.as_strided` 重建；runtime-created FP8 与 DLO 组合必须使用 no-AllGather 路径。\n- 禁止：对转置 Cutlass FP8 weight 使用逻辑 `.flatten()` 后再 `.view(shape)`，丢失物理布局；为调用幂等处理而把已量化 CPU 层重新搬回 accelerator；把 runtime-created FP8 宣称支持 sharded DLO AllGather。\n- 验收：CPU/mock 测试验证普通 layerwise、DLO prefetch 和 resident replay 的值与 stride 均保持，覆盖不同 storage span、streaming 顺序、已处理层跳过与缓存释放；online FP8 + DLO AllGather 必须 fail fast，no-AllGather 路径须完成加载并通过对应输出/资源 smoke。^[PR #5910] + +相关执行流见 [Diffusion architecture](architecture.md)；component lifecycle 见 [component lifecycle 规则](rules-component-lifecycle.md)。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 57c488492..6217831a5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -38,7 +38,7 @@ confidence: high |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1j` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2n` | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o` 见 [LoRA](rules-lora.md) | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j`（`4a`–`4i` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)） | @@ -122,127 +122,8 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：在 CRF round trip 前 resize 或 squash 图像；按模型路径名猜版本；把非零 CRF 静默应用到 tensor；将 1-pixel 维度向下取整为零后交给 libx264。 - 验收：覆盖默认/显式 CRF、PIL/tensor、无 PyAV、缺少 libx264、1xN 图像和非等比例输入，断言 CRF 顺序、错误类型、原图保护以及最终 conditioning shape。^[PR #6070] -### DIFF-2a — checkpoint remap 必须追到已注册且真实消费的目标 - -- 触发：增加或修改 weight mapper、scale 名称、quantization adapter 或 key resolution。 -- 强制：从序列化 key 追到目标 layer 注册的 parameter/buffer 和 forward consumer； - 多条 resolution path 必须返回对称、由合同解释的目标名。模型专属 grouped-QKV reorder、 - fused gate/up split 等布局变换必须在调用 active weight loader 前完成，让 quantization wrapper - 与 TP shard 接收最终语义布局。 -- 禁止：把 producer 有而当前 consumer 不支持的 tensor 静默过滤；必须 fold/map 或 - fail fast，并在错误中标明依赖的 upstream 版本边界。 -- 验收：测试覆盖已消费 key、未知 key、当前版本不支持的 scale 以及两条 resolution - path 的输出名；fused mixed-FP8 还须覆盖 weight/scale/shard、gate/up 顺序、nested serialized - FP8 和 combined projection，缺 shard 不得默认 0。 ^[PR #5087] ^[PR #5737] ^[PR #5848] -- 强制：读取 transformer-local `config.json` 时，disk method 与 active method 归一化后不一致 - 必须失败；disk 声明 serialized、`data_type=mx_fp` 或不同 `ignored_layers` 时应按该 component - 重建 config，不能用 online/default 配置解释 serialized 权重。目标 pin 的 - `resolve_quant_config_from_disk` 只有 Wan2.2 一个 live caller，且无 helper 直测；在补齐多模型 - caller 与 mismatch/serialized/ignored-layers 测试前，这只是局部 loading guard，不是全局自动 - checkpoint reconciliation 支持。^[PR #5839] - -### DIFF-2b — HSDP/FSDP 修复必须执行真实 fully_shard - -- 触发：改动 diffusion HSDP/FSDP 参数过滤、packed/scalar parameter 或 DeviceMesh。 -- 强制：至少用单 rank Gloo + CPU DeviceMesh 执行一次真实 `fully_shard`。 -- 禁止：只断言传给 mock 的 kwargs 后声称分布式语义已覆盖。 -- 验收：普通 float parameter 变为 DTensor；packed uint8/scalar parameter 保持本地 - identity，并覆盖 loader 的真实调用边界；replicate size 非正值拒绝，1 用 1D、>1 用 2D - DeviceMesh，测试 world size 与参数一致。 ^[PR #5088] ^[PR #5872] - -### DIFF-2c — component quantization 独立解析且 namespace 端到端一致 - -- 触发：diffusion pipeline 为 text encoder、transformer、VAE 等组件增加独立量化配置。 -- 强制：每个组件独立解析量化配置，并选择一个端到端一致的 namespace：要么 selector - 保留 `text_encoder` 等完整 component owner prefix 到持权 layer，要么 pipeline 先 resolve - 单一 component，再让内部 layer 使用 component-relative prefix。只量化明确支持的 linear， - embedding、LM head、patch/final projection 等排除项必须显式保持未量化。 -- 强制：`ComponentQuantizationConfig.resolve()` 只对 `get_quant_method` 收到的 runtime prefix 做 - longest-prefix match，未命中才落到 default；它不会自动感知 checkpoint `WeightsMapper`。 - 因此 mapper、pipeline 与 layer 必须显式维持同一 namespace，并测试重叠 prefix 的最长匹配。 -- 禁止：半途裁掉 owner prefix；resolve component 后又要求 ignored layer 带 owner；用一个 - 组件配置隐式覆盖其他组件；因为同属一个模型就量化全部 linear。 -- 验收：至少覆盖“只量化一个组件、其他组件保持 BF16”的真实构造与加载，逐层断言 - 命中/排除集合，并验证 meta-device parameter 不会被提前 move。FLUX.2 的具体边界见 - [FLUX.2 规则](../../models/flux2/rules.md)，component-relative 变体见 - [MiniMax H3 规则](../../models/minimax-h3/rules.md)；新增 quantization×offload 组合需给兼容矩阵， - 未验证 quantizer fail-closed，并核对 dtype/shape/stride。 ^[PR #5136] ^[PR #5737] - ^[PR #6279] - -### DIFF-2d — Artifact identity 只信任已验证的不可变来源 - -- 触发：新增或修改 host-weight artifact、warm restore、source identity、layout/dtype policy。 -- 强制：共享 identity、restore 和 model ABI 保持 representation-neutral，dtype/layout 只属于 - 具体 policy/producer；只有同时验证 source kind、snapshot revision、repo topology 和 blob - 位置的内容寻址仓库，才可用 blob 名代替内容哈希，其他本地文件和 symlink 必须按内容定身份。 -- 禁止：因 symlink 目标名看似 40/64 位十六进制就假设不可变；把 `bf16`、具体 layout 或首个 - producer 名写进共享 identity/restorer API；用进程内 inode/mtime guard 代替跨启动身份校验。 -- 验收：第二种 synthetic representation 复用共享 identity/restorer；任意本地十六进制命名 - symlink 在同大小内容被替换后产生不同 identity；合法 Hub snapshot→blob 拓扑仍走受控快捷路径。 - ^[PR #6445] - -### DIFF-2e — distributed layerwise offload 只能选择一个一致的权重分片合同 - -- 触发：新增 `enable_distributed_layerwise_offload`、`dlo_use_allgather`、模型 - `OffloadPlan` 或 block discovery。 -- 强制：明确 rank 本地 CPU shard、固定双 GPU buffer、H2D/AllGather stream 和模型 - block list 的 owner；`dlo_use_allgather=false` 保留 standard loader 已生成的每-rank - ready-to-run layout（没有其他分片维度时才是 full weight，TP 下则是 TP-local shard），开启 - AllGather 时不得再叠加已 DTensor-sharded 的 HSDP 参数。共享 output buffer 可按所有 hook - 的最大 block 分配，但每个 hook 调 collective 前必须按 dtype 截成自己的 - `_ag_output_sizes[dtype] == dp_size * local_shard.numel()` 前缀；block-flat-relative repoint - offset 只能在这段实际 block buffer 内解释。 -- 强制：AllGather 权重组复用既有 topology：DP>1 优先 DP group；DP=1 且 SP>1 才用 SP group。 - TP 不作为 DLO AllGather group；进入 mmap path 时 TP>1 必须因绕过 TP-aware weight-loader - callback 而拒绝。no-AllGather 只关闭 DLO 新增的权重 collective，不关闭 standard loader - 已建立的 TP/HSDP/SP 语义，也不提供跨 rank host-weight 节省。DP=SP=1 不能从“无 process - group”推断为可用的 rank-local AllGather 模式：目标 pin 的 loader 在 DLO+AllGather、支持 mmap - 时会跳过 `load_weights()`，但 backend 只有 `dp_size>1` 才执行 mmap load，两个 gate 不一致； - 修复并回归前单 rank 应使用 no-AllGather 或普通 layerwise offload。 -- 强制：非 DiT component 的 staging 必须由 `OffloadPlan` 显式声明：encoder block group - rank-local streaming 不借用 DiT AllGather group，on-demand component 由 pipeline 在真实 - encode/decode stage 成对 load/offload，resident layer 只对声明的 DiT path 生效。no-AllGather - 路径保留 standard loader 产生的 TP-local layout；若模型 loader 还做 QKV/MLP transform， - mmap 必须显式 opt out，直到 transform-before-shard 等价性有测试。 -- 禁止：用 heuristic 找不到 block 时静默假装 offload 已启用；对 HSDP 参数二次分片； - 把 CPU 内存、AllGather 同步和设备显存开销隐藏在一个泛化的 `enable_cpu_offload` 开关； - 把 max-sized shared buffer 原样交给较小 block 的 `all_gather_into_tensor`。 -- 验收：CPU/Gloo 单 rank 覆盖 DTensor wrapper、shard padding、双 buffer 和 disable - cleanup；配置测试覆盖 AllGather+HSDP 的明确失败及 no-AllGather 的允许路径。另以至少两个 - 不同 block size 验证 small/large hook 都满足 collective size equality、重建权重和 repoint。 - 当前回归用 mocked collective 检查 8/32-element、dp=2 合同；没有真实 multi-rank collective - 覆盖，PR 的 8×NPU H3 E2E 说明也缺少已填写的 commit 证据，不能升级为验证结论。 ^[PR #5802] - DLO DP collective-wave admission 见 [DIFF-2k](rules-output-lifecycle.md)；no-AllGather 的 - TP-local request 不得误走 fused batch。文档 compatibility matrix 中 - “accepted” 默认只代表配置/源码 guard；PR #5836 的 predecessor partition-path commit 为 H3 - USP4+HSDP4+no-AllGather+8 resident layers 提供一次 4×H100、3/3 completion/perf run，故该 exact - 组合不再是纯 config evidence，但没有 ordinary-offload 数值/输出质量对照，且 profiler 开启、 - final repo-root modular route 未复测，仍不得升级为 production support。TP+no-AllGather、一般 - DP+SP 等其他组合仍缺完整 model×hardware E2E。^[PR #5764] ^[PR #5839] ^[PR #5836] - -多 DiT component discovery 与 dotted-path lifecycle 合同见 -[DIFF-2f–2j 专页](rules-component-lifecycle.md)。 - ## 质量阈值与资源辅助 -### DIFF-2p — HSDP/FSDP2 必须同时识别 legacy 与 online FP8 linear method - -- 触发：修改 diffusion HSDP/FSDP2 参数准备、online FP8 quantization method 或非连续权重布局处理。 -- 强制：同时识别 `Fp8LinearMethod` 与 `Fp8PerTensorOnlineLinearMethod`；当 FP8 权重是转置形成的非连续 view 时，先将底层 `(out, in)` row-major storage 替换为 contiguous weight 供 FSDP2 接受，再让 kernel 的 `_get_layer_params` 返回对应转置 view，且同一 kernel 只 patch 一次。 -- 禁止：只检查 legacy `Fp8LinearMethod`、把非连续 transpose view 直接交给 `fully_shard`，或因新版 online method 已被识别就宣称所有 FP8 method 和布局均兼容。 -- 验收：legacy 与 `Fp8PerTensorOnlineLinearMethod` 都覆盖 regression test，断言重写层数、weight contiguous、`(out, in)` shape，以及 kernel 返回的转置 view shape/stride；另按 HSDP/FSDP 规则执行真实 `fully_shard` 验证。^[PR #5677] - -### DIFF-2q — LTX checkpoint profile 必须贯穿 revision 与 transformer 子目录 - -- 触发：新增 LTX checkpoint/version profile、Hub `revision`、Full/Distilled transformer 配置或组件加载路径。 -- 强制：显式 `model_version` 优先于结构启发式；`revision` 必须贯穿 metadata detection、prefetch、tokenizer、全部组件、transformer config/weight source、scheduler 和 post-process sample-rate；LTX-2.5 Full/SFT 使用 `transformer_full`，distilled 与旧版本使用 `transformer`，并在缺少 Gemma4 时提示 `transformers>=5.10.1,<5.15`。 -- 禁止：按路径名识别版本；只给 config lookup 传 revision 而让权重加载回到 HEAD；用 `transformer/config.json` 解释 Full checkpoint；把 transient metadata/Hub 异常静默降级为 LTX-2 并宣称 profile 已确定。 -- 验收：固定 revision 的 converted/official split 链路逐项断言所有 loader 调用和输出 metadata；用两个 transformer config marker 验证 Full 子目录选择，覆盖 local/HF、legacy/distilled/Full profile 及缺失 encoder 版本错误。^[PR #6070] - -### DIFF-2s — 在线量化加载与 layerwise offload 必须保留物理权重布局 - -- 触发：修改在线/runtime quantization 与 layerwise offload、DLO 权重加载、非连续 FP8 权重 flatten/repoint 或加载后处理流程。\n- 强制：在线量化层必须在消费完对应 checkpoint weight 后才可流式移回 CPU，并跳过已完成层的重复 `process_weights_after_loading`；继续加载其他模型权重前释放可复用的量化缓存。对非连续权重按物理 storage 顺序打包，记录 `shape`、`stride` 和实际 storage span，并在普通 layerwise、DLO prefetch 与 resident replay 中用 `torch.as_strided` 重建；runtime-created FP8 与 DLO 组合必须使用 no-AllGather 路径。\n- 禁止：对转置 Cutlass FP8 weight 使用逻辑 `.flatten()` 后再 `.view(shape)`，丢失物理布局；为调用幂等处理而把已量化 CPU 层重新搬回 accelerator；把 runtime-created FP8 宣称支持 sharded DLO AllGather。\n- 验收：CPU/mock 测试验证普通 layerwise、DLO prefetch 和 resident replay 的值与 stride 均保持，覆盖不同 storage span、streaming 顺序、已处理层跳过与缓存释放；online FP8 + DLO AllGather 必须 fail fast，no-AllGather 路径须完成加载并通过对应输出/资源 smoke。^[PR #5910] - ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md index 3c9753ea8..579b72660 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md @@ -1,7 +1,7 @@ --- title: "Host Weight Runtime" created: 2026-08-23 -updated: 2026-08-23 +updated: 2026-09-04 type: index tags: [vllm-omni, components] sources: ["PR #6419", vllm_omni/host_weight_runtime/] @@ -17,7 +17,7 @@ confidence: high ## 什么时候查这里 - 修改 host-weight store、lease/lock、deny/quarantine、locality、fallback 或 restore transaction。 -- 具体 diffusion layout/source identity 同时读取 [Diffusion DIFF-2d](../diffusion/rules.md)。 +- 具体 diffusion layout/source identity 同时读取 [Diffusion DIFF-2d](../diffusion/rules-checkpoint-loading.md)。 ## 目录内容 diff --git a/knowledge/repos/vllm-omni/docs/design-doc-map.md b/knowledge/repos/vllm-omni/docs/design-doc-map.md index bb71bfff7..115ea030e 100644 --- a/knowledge/repos/vllm-omni/docs/design-doc-map.md +++ b/knowledge/repos/vllm-omni/docs/design-doc-map.md @@ -1,7 +1,7 @@ --- title: "官方设计文档地图（docs/design/**）" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-04 type: guide tags: [vllm-omni, docs] sources: ["PR #5139", "PR #5833", "PR #5839", "PR #5914", "PR #5958", "PR #6045", docs/features/README.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, .github/CODEOWNERS, docs/.nav.yml, docs/design/index.md, docs/design/module/, docs/design/module/archive/README.md, docs/design/ar_diffusion_pipeline_capability.md, docs/design/architecture_overview.md, docs/design/feature/diffusion_continuous_batching.md, docs/design/feature/distributed_layerwise_offload.md, docs/design/feature/omni_async_output_materialization.md, docs/design/feature/quantization.md, docs/user_guide/diffusion/execution_modes.md] @@ -57,8 +57,8 @@ Feature user guide、design contract、recipe 与 support claim 的分层和 nav | `feature/async_chunk.md` | 跨 stage 分块流式 | [distributed](../components/distributed/async-chunk.md) | | `feature/cache_dit.md`、`teacache.md`、`prefix_caching.md` | 缓存加速 | [diffusion](../components/diffusion/cache-acceleration.md) | | `feature/{tensor,pipeline,sequence,expert,cfg,vae}_parallel.md`、`hsdp.md` | 并行策略 | [diffusion](../components/diffusion/parallelism.md) | -| `feature/distributed_layerwise_offload.md` + `user_guide/diffusion/cpu_offload.md` | DLO 的 DP/SP/TP/HSDP compatibility、AllGather 与 rank-local loader 合同；部分组合仅有配置级验证 | [diffusion rule DIFF-2e](../components/diffusion/rules.md) | -| `feature/quantization.md` + `user_guide/quantization/overview.md` | diffusion quantization factory、component routing、checkpoint metadata 与验证边界 | [diffusion rule DIFF-2c](../components/diffusion/rules.md) | +| `feature/distributed_layerwise_offload.md` + `user_guide/diffusion/cpu_offload.md` | DLO 的 DP/SP/TP/HSDP compatibility、AllGather 与 rank-local loader 合同；部分组合仅有配置级验证 | [diffusion rule DIFF-2e](../components/diffusion/rules-checkpoint-loading.md) | +| `feature/quantization.md` + `user_guide/quantization/overview.md` | diffusion quantization factory、component routing、checkpoint metadata 与验证边界 | [diffusion rule DIFF-2c](../components/diffusion/rules-checkpoint-loading.md) | | `metrics.md`、`qwen3_omni_tts_performance_optimization.md` | Prometheus 指标；TTS 性能优化实录 | [qwen-omni](../models/qwen-omni/architecture.md)（perf 部分） | | `docs/configuration/*` | 配置 schema spec | [configuration](../components/configuration/architecture.md) | | `docs/contributing/ci/*` | L1–L5 与 markers | [ci guides](../ci/guides/test-tiers.md) | From 239e4de5ba649ba7cbf19f639409c93b9d5534f8 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:32:52 +0800 Subject: [PATCH 193/465] knowledge(vllm-omni): sync 3c6648d8 (PR #6293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 3c6648d8a54902297ca7d46815e9883c1de6b204 Upstream-PR: #6293 Knowledge-Delta: 新增配置公开导出的延迟加载与 NPU 模型补丁 runtime 时序规则，记录 PR #6293 修复的 DiffusionOutput 导入回环。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-platform-backends.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index fba805b6c..b3c0b84fd 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2fdbf2234aeb76715618a5b236f0016f115e3e64 + previous_audited_sha: d7216ef5144866c2f039947e1cc396b1984fcf09 audited_ref: main - audited_sha: d7216ef5144866c2f039947e1cc396b1984fcf09 + audited_sha: 3c6648d8a54902297ca7d46815e9883c1de6b204 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a21d1f604..35a03bb60 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d7216ef5144866c2f039947e1cc396b1984fcf09`. The canonical machine baseline +3c6648d8a54902297ca7d46815e9883c1de6b204`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 38e1f0ce1..6cef29151 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d7216ef5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3c6648d8` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index fe2a317b6..b3dc35905 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293"] --- # vLLM-Omni 配置开发门禁 @@ -263,3 +263,10 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 强制：通过 `is_runai_obj_uri` 将对象存储模型的 `*.model`、`*.py`、`*.json` 轻量文件按 URI 缓存一次到确定性的 `model_streamer/` 目录；`get_config`、`config.json` 和 `model_index.json` 的读取统一使用该本地目录，同时让各 stage 继续接收原始 URI 以保留 `model_weights` 的流式加载语义。 - 禁止：把原始对象存储 URI 交给 Hugging Face 仓库解析；把临时 `model_streamer/` 路径传播成 stage 的模型身份；用完整 URI、bucket 或组织名参与模型类型 basename 匹配。 - 验收：用 mock 覆盖对象存储 URI 的单次物化、配置驱动的 pipeline 选择、`model_index.json` 回退和 basename-only 匹配，并确认欺骗性 bucket 名不能劫持 pipeline；真实对象存储运行另行验证。^[PR #5036] + +### VOMNI-CFG-2a — 配置公开导出必须延迟加载以保持导入链无环 + +- 触发：修改 `vllm_omni.config` 的公开导出，或新增会经 pipeline registry、模型 pipeline 与 diffusion 数据模块回环的配置导入路径。 +- 强制：对会 eagerly 导入 `pipeline_registry`、`StageConfigFactory` 或其他重型 pipeline 依赖的公开配置符号使用模块级延迟解析；首次访问时通过 `importlib` 解析并缓存真实对象，同时保持 `__all__`、`__dir__` 与直接子模块导入的身份一致。导入链测试必须在干净解释器中先验证轻量配置导入不会预加载 registry/PI0，再验证公开导出仍解析到直接实现。 +- 禁止：在 `vllm_omni.config` 包初始化阶段无条件导入会反向加载 `DiffusionOutput` 的 registry；用 import 顺序或已预加载模块掩盖循环；删除公开导出或让 lazy proxy 与直接导入得到不同对象。 +- 验收：静态检查顶层导入不含重型 registry/factory，干净子进程按 `vllm_omni.config.lora` → `vllm_omni.diffusion.data` 顺序导入成功，并断言 `StageConfigFactory`、`register_pipeline` 与直接模块导出的对象相同；覆盖仍需用这些公开符号的调用方。^[PR #6293] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a6c6f0391..ff37a0069 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d7216ef5` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 3c6648d8` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 843424db5..917dbe7a7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d7216ef5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3c6648d8` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index fa85cee84..bb1a1d5bd 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d7216ef5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3c6648d8` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index fcaf4cae1..8ef976099 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604"] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293"] confidence: high --- @@ -42,3 +42,10 @@ confidence: high - 禁止：捕获整个请求编排或包含动态 Python/RNG 状态的路径；使用无界 shape cache、绕过 global graph pool、在 nested capture 中重放、复用会被下一次重放覆盖的输出，或吞掉 capture、patch setup、SDPA context 的异常后继续运行。 - 验收：CPU/mock 测试覆盖 exact-signature dispatch、cached/uncached bucket、global graph pool、persistent-output ownership、capture failure fail-stop 和异常传播；A3 硬件测试分别比较 eager 与 uncached/cached NPUGraph 的输出、shape 和 cache，并核对图数量上限与 runtime preconditions。^[PR #5604] +## EXEC-13c — NPU 模型补丁必须避开平台构造期的 diffusion 导入回环 + +- 触发：NPU 平台注册模型专有 monkey patch，且该 patch 包会导入 diffusion pipeline/data 或依赖已完成初始化的平台状态。 +- 强制：平台构造阶段只完成轻量、全局且无 diffusion 回环的 patch；将会加载模型 encoder/pipeline 的 patch 延迟到 `init_diffusion_model_runner_runtime`，并在 diffusion pipeline 加载前、平台对象已建立后，按既有顺序显式应用所有相关 patch。注册入口仍须幂等且只作用于目标 consumer。 +- 禁止：在 NPU platform `__init__` 中导入或执行会经 `pipeline_registry` → `DiffusionOutput` 回环的模型 patch；依赖 import side effect；只移动一个 patch 而让同一模型的相关 patch 仍在过早阶段加载；把 NPU 时序修复外推为所有平台的通用行为。 +- 验收：静态或子进程测试证明平台构造不会加载 MiniMax-H3 encoder/diffusion data，runtime hook 在 pipeline 加载前调用目标 patch 且保留其他 patch；验证重复初始化不重复 patch，并用真实 NPU consumer 覆盖 patch 生效路径。^[PR #6293] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a2839bf6d..671fea92c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d7216ef5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3c6648d8` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 58925874f..edbca49a6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d7216ef5` 验证存在 +- 源码校验：以上路径均已在 `main @ 3c6648d8` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5ad82cc6e..fdf6741e5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d7216ef5`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3c6648d8`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `2fdbf223` 本轮记录 PR #4499 为非流式 `/v1/audio/speech` 原始音频响应增加 token usage headers 的 serving 规则；文档更新和测试变更不纳入组件知识。不能据此推断 streaming raw-audio 响应支持同样 headers，或 diffusion-mode speech server 会发出这些 headers。 +本次相对 `d7216ef5` 本轮将配置包的延迟导出与 NPU 模型补丁时序写入现有组件规则；测试文件归 CI/testing owner，未写入规则页。不能据此推断所有平台或所有模型都存在相同的导入循环，或仅凭测试通过就证明完整 NPU 启动链路已覆盖。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 2fdbf223 \ - --to d7216ef5 \ + --from d7216ef5 \ + --to 3c6648d8 \ --repo \ --mode report-only ``` From 28ee3e1d6463bd04d16800ca5e661404248a9390 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:48:40 +0800 Subject: [PATCH 194/465] knowledge(vllm-omni): split the parallel-topology series out of configuration rules configuration/rules.md had 1293 bytes of headroom with 46 remaining units routing to it, so a single new CONF rule would have crossed the 32768-byte gate and halted a unit mid-flight. split_risk.py had been naming it; this lands the split ahead of that, stopping at a unit boundary through the PAUSE file rather than racing it. Two series, extracted whole rather than by byte count: CONF-4a-4d -> rules-parallel-topology.md composable strategy axis, new parallel axes reaching the final topology, headless topology parity, and the pipeline-wide load balancing owner CONF-5a-5g -> rules-topology-profiles.md frozen topology, auxiliary stage injection, model-specific deploy profiles Rule IDs unchanged, both new pages registered once in _index.md, and the two group rows that named the moved rules -- deploy-topology and composable-strategy -- now name the page each rule went to rather than the page it left. That is the failure the diffusion split shipped and review caught: a routing row that still resolves, sending reviewers to a document the rule is not on. Three frontmatter errors of my own, all from the extraction template. Two were caught by wiki lint: the sources list is built by joining the PRs cited in the extracted span, and parallel-topology cites none, so it emitted a leading empty element in the flow sequence -- not valid YAML, so the whole block failed to parse and lint reported the page as having no frontmatter at all; and the tag was written as "configuration", the directory name, where the SCHEMA taxonomy uses "config", which is what the parent page carries. The third no validator can see, and review caught it: the template also guesses a source path, and it guessed vllm_omni/config/parallel.py, which has never existed -- the composable strategy code lives in vllm_omni/config/composable_parallel/. A rules page whose only declared provenance is a file that is not in the tree is worse than one with none, because it reads as verifiable. The sources now name the three paths the parent page routes these rules to, each confirmed present at the pinned SHA, and the same check was run against the other three guessed paths in this pair of splits (vllm_omni/deploy/, entrypoints/openai/, inputs/) -- those are real. Deriving the paths from the routing table instead of guessing is the fix the template still needs. No unit is consumed and audited_sha is untouched. Gates: tree 0 errors / 14 reminders, wiki lint 0 errors across 337 pages, release audit CLEAN, 107 anchored links clean. Headroom after: rules.md 11037 B. Co-Authored-By: Claude Opus 5 (1M context) --- .../components/configuration/_index.md | 4 +- .../configuration/rules-parallel-topology.md | 58 ++++++++++ .../configuration/rules-topology-profiles.md | 71 ++++++++++++ .../components/configuration/rules.md | 103 +----------------- 4 files changed, 134 insertions(+), 102 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/configuration/rules-parallel-topology.md create mode 100644 knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6cef29151..ed375db78 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni Configuration" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-04 type: index tags: [vllm-omni, components, config] sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] @@ -43,3 +43,5 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] | 核对 pipeline/deploy schema | [pipeline deploy schema](pipeline-deploy-schema.md) | | 计算多 stage 显存预算 | [deployment memory budget](deployment-memory-budget.md) | | 添加新模型和注册点 | [adding a model](adding-a-model.md) | +| composable strategy 的 axis、并行拓扑与 load balancing owner | [并行拓扑合同](rules-parallel-topology.md) | +| 冻结 topology、辅助 stage 注入与模型专用 deploy profile | [topology 与部署 profile](rules-topology-profiles.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-parallel-topology.md b/knowledge/repos/vllm-omni/components/configuration/rules-parallel-topology.md new file mode 100644 index 000000000..0f02e93af --- /dev/null +++ b/knowledge/repos/vllm-omni/components/configuration/rules-parallel-topology.md @@ -0,0 +1,58 @@ +--- +title: "并行拓扑合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, config] +sources: [vllm_omni/config/composable_parallel/, vllm_omni/config/config_factory.py, vllm_omni/config/stage_config.py] +confidence: high +--- + +# 并行拓扑合同 + +`CONF-4a`–`CONF-4d`：composable strategy 的 axis 暴露、新并行轴进入最终拓扑、标准与 headless 启动的拓扑一致性，以及 strategy/CLI/pipeline-wide load balancing 的唯一 owner。触发条件与其余审查组见 [Configuration 规则](rules.md) 的 Direct 代码快速入口。 + +## CONF-4a — composable strategy 只暴露已经接通的 axis + +- 触发：新增 composable parallel strategy、axis、routing/LB policy 或 stage override。 +- 强制：schema 明确区分 wired 与 reserved；translator 对 unsupported/reserved 值显式 + 失败。routing 与 load-balancing policy 只能有一个 owner。 +- 禁止：接受但静默忽略 axis；关键 strategy/deploy 文件缺失时 conditional skip；以 + 可漂移的 stage index 作为长期 identity。 +- 验收：每个公开 axis 都有 spec → translator → final stage config 的正向测试和 + unsupported fail-fast 测试；stage 使用稳定名称。 + +## CONF-4b — 标准与 headless 启动必须解析出同一拓扑 + +- 触发：CLI、headless serve、offline entrypoint 或 engine factory 新增/转发配置字段。 +- 强制：同一命令语义在所有入口转发相同 override，并比较展开后的逐 stage 配置。 +- 禁止：只测 parser 输出，未证明值到达 engine/config consumer。 +- 验收：标准与 headless 路径对同一 override 生成等价 topology；缺字段时直接失败， + 不允许用 skip 隐藏路径漂移。 + +## CONF-4c — 新并行轴必须进入最终拓扑并由 consumer 校验 + +- 触发：新增 diffusion parallel axis、text-encoder TP、CFG/patch parallel 或把 + `strategy-config` 的声明映射到 stage runtime。 +- 强制：从 strategy spec/CLI 到 `OmniStageDiffusionParallelConfig` 和最终 process + group 展开完整链路；记录 `data × cfg × sequence × pipeline × tensor` 的设备乘积， + 并由实际 consumer 校验 rank/head divisibility、模型不支持的组合和可见设备容量。 +- 禁止：parser 接受字段后在 projection、engine kwargs 或 worker 初始化时丢失；用一个 + generic upper bound 替代模型自己的约束；用 recipe 示例或单纯 dataclass 构造代替真实 + topology/consumer 证据。 +- 验收：正向测试断言新值可从最终 stage config 读回；负向测试覆盖不整除的 head/rank、 + `cfg_parallel_size` 不适用的模型和设备不足，并在 worker 创建前失败。 + +## CONF-4d — strategy、CLI 与 pipeline-wide load balancing 必须保留唯一 owner + +- 触发：`strategy_config`、legacy YAML、`--omni-lb-policy` 或 stage replica/load + balancing 同时提供策略。 +- 强制：registry-backed strategy 才进入 translator；legacy 路径明确 warning 并保持 + 原语义；合并顺序固定为 deploy/pipeline → strategy → CLI，CLI 的显式冲突必须可见， + pipeline-wide load-balancer 只由 orchestrator 拥有并传入各 stage。 +- 禁止：接受 legacy `strategy_config` 后静默声称已生效；把 CLI 的默认值误当显式覆盖； + 每个 stage 各自构造一个相互冲突的全局 load balancer。 +- 验收：覆盖 registry、legacy、CLI 非默认值和冲突值，断言最终拓扑、设备乘积和单一 + load-balancer consumer；策略生效前若设备不足必须 fail fast。 + +部署 YAML 的展开顺序见 [deploy-yaml](deploy-yaml.md)；stage 执行合同见 [model-executor 规则](../model-executor/rules.md)。 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md new file mode 100644 index 000000000..8eaa7289e --- /dev/null +++ b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md @@ -0,0 +1,71 @@ +--- +title: "topology 与部署 profile 合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, config] +sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #6082", vllm_omni/deploy/] +confidence: high +--- + +# topology 与部署 profile 合同 + +`CONF-5a`–`CONF-5g`：冻结 topology 的单一来源、可选辅助 stage 注入、缺少 model_type 的多 stage checkpoint 路由，以及模型专用 deploy profile 的一致传递。触发条件与其余审查组见 [Configuration 规则](rules.md) 的 Direct 代码快速入口。 + +## CONF-5a — 冻结 topology 只保留一份，部署开关决定 wiring + +- 触发：同一模型的 sync/async processor、普通/async chunk 或部署加速出现多份近似 + pipeline/stage YAML。 +- 强制：相同冻结拓扑共用一份 pipeline config，同时声明可选 processor；由 deploy + flag 决定 wiring。模型 package `__init__` 保持轻量，FP8 等部署加速留在 deploy 层。 +- 禁止：为一个 runtime flag 复制整套 topology；把无关 payload bug 或平台 cleanup + 混入 config migration。 +- 验收：两种 wiring 都从同一 topology 展开，配置 diff 只包含预期 deploy 字段； + 同卡多 stage 的 `gpu_memory_utilization` 总预算不超过可用容量。 + +## CONF-5b — `trust_remote_code` 的未指定值必须保留 deploy 优先级 + +- 触发：CLI、structured factory、legacy stage factory 或 deploy YAML 同时提供 + `trust_remote_code`。 +- 强制：调用方显式 `True`/`False` 才覆盖 per-stage deploy 值；未指定用 `None` 表示并 + 保留 YAML 值，HF config resolution 才把 `None` 收敛为安全的实际 bool。 +- 禁止：把 `store_true` 的 absent-False 当成显式 False 无条件写入 override；structured + 与 legacy 各自实现 precedence；在末端用默认值静默覆盖用户明确的 False。 +- 验收：覆盖“未指定、显式 True、显式 False”三态，并从 structured/legacy 两条路径 + 断言最终 stage config；`with_trust_remote_code_override` 是唯一合并入口。 + +## CONF-5c — 可选辅助 pooling stage 必须在最终 topology 中显式注入 + +- 触发：为现有 pipeline 增加可选辅助 stage，或新增 `--forced-aligner`、`--forced-aligner-config`、`--forced-aligner-device` 这类只由 orchestrator 读取的开关。 +- 强制：在最终 merge 前把 CLI/YAML 解析为一个 canonical stage config；任一 aligner 来源生效时复制 pipeline/deploy，追加连接到前一 stage 的 `runner="pooling"` 终端 stage，并将 `default_pooling_params.task`、设备和 decoder hook 投影到该 stage；无来源时保持原 topology。 +- 禁止：只检查 `forced_aligner` 而忽略 config-only 注入；原地修改调用方的 pipeline/deploy；把 orchestrator-only flags 泄漏为每个 stage 的 engine 参数；用 `execution_type` 猜测应构造 `PoolingParams` 还是生成参数。 +- 验收：覆盖模型参数、config-only、无参数三态；断言原 pipeline/deploy 未改变，最终逐 stage config 含正确模型、`runner`、pooling task、设备和 decoder hook，并证明 stage 初始化按 `runner` 选择 `PoolingParams`。 +^[PR #4795] + +## CONF-5d — 无 `model_type` 的多 stage checkpoint 必须闭合别名与架构路由 + +- 触发：接入缺少 `model_type` 或 `architectures` 的 NeMo 风格多 stage checkpoint，或新增其 pipeline alias、architecture 路由与默认 deploy 配置。 +- 强制：同时登记所有 stage architecture 到 `_ARCH_TO_MODEL_TYPE`，注册统一 `NemotronVoiceChatConfig`，让 alias、path-basename fallback 与 `default_deploy_config_name` 指向同一 pipeline；不得只依赖路径猜测。 +- 禁止：只注册 thinker 或只提供一个 alias；让空的 `model_arch` 覆盖 checkpoint architecture；让 sync 与 async deploy 配置解析出不同的 stage 拓扑。 +- 验收：用 bare `config.json` 和三个 architecture 分别验证 model type、pipeline、stage config 与默认 deploy 名称；sync/async 配置只切换 processor wiring，并保持三 stage 顺序与 stage architecture 一致。 +^[PR #5842] + +## CONF-5e — MiniCPM-o 部署 profile 必须一致传递 CFM Graph 开关 + +- 触发：修改 MiniCPM-o 4.5 bundled deploy YAML 或 Code2Wav connector extra 中的 CFM CUDA Graph 开关、缓存上限或配置传递。\n- 强制：`minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml`、`minicpmo_4_5_3gpu.yaml` 和 `minicpmo_4_5_8x4090.yaml` 必须一致声明 `enable_cfm_graph` 与 `cfm_max_graphs`；当前默认值为 `true` 与 `32`，并传入 `MiniCPMO45Code2Wav` 的 `_cfm_graph_config` 和 `BatchedToken2Wav`。非 CUDA 设备仍必须回退 eager。\n- 禁止：只更新一个 deploy profile、依赖未记录的默认值，或把配置存在误认为 graph 已在非 CUDA 环境生效；不得把模型专有开关扩展成所有模型的通用配置合同。\n- 验收：解析四份 deploy 配置并断言开关和上限一致，追踪非默认值到 `_cfm_graph_config` 与 `BatchedToken2Wav`；覆盖开关关闭和非 CUDA fallback，并确认 stage topology 未发生额外变化。\n^[PR #6082] + +## CONF-5f — π0 的 checkpoint、registry 与 deploy 必须闭合为单 diffusion stage + +- 触发：新增或修改 π0 的 `pipeline_registry.py`、`vllm_omni/deploy/pi0.yaml`、`OmniDiffusionConfig` 的 `type: pi0` autodetect，或 `Pi0Pipeline` 的 stage declaration。 +- 强制：让 `pipeline: pi0`、`OMNI_PIPELINES["pi0"]`、`Pi0Pipeline` 与 checkpoint `type: pi0` resolver 闭合；deploy 配置只能声明一个 diffusion stage，最终输出类型为 `action`，并让 `policy_server_config` 的 `image_resolution`、`action_horizon`、`action_dim`、camera 数量与 `model_config` 保持一致。 +- 禁止：只凭 checkpoint 路径或文件存在推断 pipeline，额外添加 model-executor 层或多 stage topology，或让 OpenPI metadata 与最终 stage/model config 的动作尺寸不一致。 +- 验收：无显式 deploy 配置的 checkpoint autodetect 能解析到 `Pi0Pipeline`；解析最终逐 stage config 断言只有一个 diffusion stage、`final_output_type=action`，并核对 `pi0.yaml` 与 websocket handshake metadata 的关键字段一致。^[PR #4222] + +## CONF-5g — MiniCPM-o NPU Code2Wav 开关必须按 Stage 2 传递 + +- 触发：修改 MiniCPM-o 4.5 bundled deploy YAML，或新增/调整 `code2wav_enable_npu_graph`、`code2wav_max_npu_graphs` 这类 Ascend Code2Wav graph 配置。 +- 强制：`minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml` 和 `minicpmo_4_5_3gpu.yaml` 必须在 `platforms.npu.stages` 中按 stage 声明配置；Stage 0/1 保持 `compilation_config.cudagraph_mode: PIECEWISE`，Stage 2 只通过 `additional_config` 传递 graph 开关和上限，默认值为 `true` 与 `32`，并保持最终 runner 的 outer eager 语义。所有断言必须基于 `_apply_platform_overrides` 与 `merge_pipeline_deploy` 展开的最终 stage config。 +- 禁止：把 Code2Wav NPU 开关放入 shared connector extra、全局配置或其他 stage；只修改一个 profile；以 YAML 原文或字段存在推断配置已到达 Code2Wav consumer；用全局 eager override 破坏 Stage 0/1 的 `PIECEWISE` 配置。 +- 验收：逐一解析三个 profile，断言 connector extra 不含两个 Code2Wav key，NPU 展开结果包含 Stage 0/1 的 `PIECEWISE`、Stage 2 的 `enforce_eager: true`、`code2wav_enable_npu_graph: true` 和 `code2wav_max_npu_graphs: 32`，并覆盖开关关闭和缓存上限为 0 的路径。^[PR #5604] + +部署 YAML 的展开顺序见 [deploy-yaml](deploy-yaml.md)；stage 执行合同见 [model-executor 规则](../model-executor/rules.md)。 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index b3dc35905..7c2d0b087 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -31,8 +31,8 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `core` | 每次配置审查 | `VOMNI-CFG-1b`, `VOMNI-CFG-1c` | | `strict-normalization` | schema、alias、unknown field、structured/legacy/direct 路径 | `VOMNI-CFG-1a`, `VOMNI-CFG-1b`, `VOMNI-CFG-1c`, `VOMNI-CFG-1d`, `VOMNI-CFG-1e`, `VOMNI-CFG-1f`, `VOMNI-CFG-1g`, `VOMNI-CFG-1h` | | `deploy-memory` | 显存预算、KV pin、多 stage 共卡 | `CONF-1a`, `CONF-2a` | -| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`, `CONF-4b`, `CONF-5a` | -| `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a` | +| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5g` 见 [topology 与部署 profile](rules-topology-profiles.md) | +| `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a`, `CONF-4c`–`4d`，见 [并行拓扑合同](rules-parallel-topology.md) | | `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `VOMNI-CFG-0a`, `VOMNI-CFG-0b` | @@ -151,105 +151,6 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：只注册 pipeline 或放置 YAML 就依赖 `model_type` 推断；用 `hf_overrides.model_type` 替代 stage architecture 路由；要求离线入口额外传参来弥补默认配置未接通。 - 验收：真实无显式 deploy 配置的初始化能读回最终 pipeline 与 architecture，backbone `get_text_config()` 仍解析为原模型类型，并覆盖 registry、默认 YAML 与最终 stage config 的一致性。 ^[PR #5666] -### CONF-4a — composable strategy 只暴露已经接通的 axis - -- 触发：新增 composable parallel strategy、axis、routing/LB policy 或 stage override。 -- 强制：schema 明确区分 wired 与 reserved；translator 对 unsupported/reserved 值显式 - 失败。routing 与 load-balancing policy 只能有一个 owner。 -- 禁止：接受但静默忽略 axis；关键 strategy/deploy 文件缺失时 conditional skip；以 - 可漂移的 stage index 作为长期 identity。 -- 验收：每个公开 axis 都有 spec → translator → final stage config 的正向测试和 - unsupported fail-fast 测试；stage 使用稳定名称。 - -### CONF-4c — 新并行轴必须进入最终拓扑并由 consumer 校验 - -- 触发：新增 diffusion parallel axis、text-encoder TP、CFG/patch parallel 或把 - `strategy-config` 的声明映射到 stage runtime。 -- 强制：从 strategy spec/CLI 到 `OmniStageDiffusionParallelConfig` 和最终 process - group 展开完整链路；记录 `data × cfg × sequence × pipeline × tensor` 的设备乘积， - 并由实际 consumer 校验 rank/head divisibility、模型不支持的组合和可见设备容量。 -- 禁止：parser 接受字段后在 projection、engine kwargs 或 worker 初始化时丢失；用一个 - generic upper bound 替代模型自己的约束；用 recipe 示例或单纯 dataclass 构造代替真实 - topology/consumer 证据。 -- 验收：正向测试断言新值可从最终 stage config 读回；负向测试覆盖不整除的 head/rank、 - `cfg_parallel_size` 不适用的模型和设备不足，并在 worker 创建前失败。 - -### CONF-4d — strategy、CLI 与 pipeline-wide load balancing 必须保留唯一 owner - -- 触发：`strategy_config`、legacy YAML、`--omni-lb-policy` 或 stage replica/load - balancing 同时提供策略。 -- 强制：registry-backed strategy 才进入 translator；legacy 路径明确 warning 并保持 - 原语义；合并顺序固定为 deploy/pipeline → strategy → CLI，CLI 的显式冲突必须可见， - pipeline-wide load-balancer 只由 orchestrator 拥有并传入各 stage。 -- 禁止：接受 legacy `strategy_config` 后静默声称已生效；把 CLI 的默认值误当显式覆盖； - 每个 stage 各自构造一个相互冲突的全局 load balancer。 -- 验收：覆盖 registry、legacy、CLI 非默认值和冲突值，断言最终拓扑、设备乘积和单一 - load-balancer consumer；策略生效前若设备不足必须 fail fast。 - -### CONF-5b — `trust_remote_code` 的未指定值必须保留 deploy 优先级 - -- 触发：CLI、structured factory、legacy stage factory 或 deploy YAML 同时提供 - `trust_remote_code`。 -- 强制：调用方显式 `True`/`False` 才覆盖 per-stage deploy 值；未指定用 `None` 表示并 - 保留 YAML 值，HF config resolution 才把 `None` 收敛为安全的实际 bool。 -- 禁止：把 `store_true` 的 absent-False 当成显式 False 无条件写入 override；structured - 与 legacy 各自实现 precedence；在末端用默认值静默覆盖用户明确的 False。 -- 验收：覆盖“未指定、显式 True、显式 False”三态，并从 structured/legacy 两条路径 - 断言最终 stage config；`with_trust_remote_code_override` 是唯一合并入口。 - -### CONF-4b — 标准与 headless 启动必须解析出同一拓扑 - -- 触发：CLI、headless serve、offline entrypoint 或 engine factory 新增/转发配置字段。 -- 强制：同一命令语义在所有入口转发相同 override，并比较展开后的逐 stage 配置。 -- 禁止：只测 parser 输出，未证明值到达 engine/config consumer。 -- 验收：标准与 headless 路径对同一 override 生成等价 topology；缺字段时直接失败， - 不允许用 skip 隐藏路径漂移。 - -### CONF-5a — 冻结 topology 只保留一份，部署开关决定 wiring - -- 触发：同一模型的 sync/async processor、普通/async chunk 或部署加速出现多份近似 - pipeline/stage YAML。 -- 强制：相同冻结拓扑共用一份 pipeline config，同时声明可选 processor；由 deploy - flag 决定 wiring。模型 package `__init__` 保持轻量，FP8 等部署加速留在 deploy 层。 -- 禁止：为一个 runtime flag 复制整套 topology；把无关 payload bug 或平台 cleanup - 混入 config migration。 -- 验收：两种 wiring 都从同一 topology 展开，配置 diff 只包含预期 deploy 字段； - 同卡多 stage 的 `gpu_memory_utilization` 总预算不超过可用容量。 - -### CONF-5c — 可选辅助 pooling stage 必须在最终 topology 中显式注入 - -- 触发：为现有 pipeline 增加可选辅助 stage，或新增 `--forced-aligner`、`--forced-aligner-config`、`--forced-aligner-device` 这类只由 orchestrator 读取的开关。 -- 强制：在最终 merge 前把 CLI/YAML 解析为一个 canonical stage config；任一 aligner 来源生效时复制 pipeline/deploy，追加连接到前一 stage 的 `runner="pooling"` 终端 stage，并将 `default_pooling_params.task`、设备和 decoder hook 投影到该 stage；无来源时保持原 topology。 -- 禁止：只检查 `forced_aligner` 而忽略 config-only 注入；原地修改调用方的 pipeline/deploy；把 orchestrator-only flags 泄漏为每个 stage 的 engine 参数；用 `execution_type` 猜测应构造 `PoolingParams` 还是生成参数。 -- 验收：覆盖模型参数、config-only、无参数三态；断言原 pipeline/deploy 未改变，最终逐 stage config 含正确模型、`runner`、pooling task、设备和 decoder hook，并证明 stage 初始化按 `runner` 选择 `PoolingParams`。 -^[PR #4795] - -### CONF-5d — 无 `model_type` 的多 stage checkpoint 必须闭合别名与架构路由 - -- 触发：接入缺少 `model_type` 或 `architectures` 的 NeMo 风格多 stage checkpoint，或新增其 pipeline alias、architecture 路由与默认 deploy 配置。 -- 强制：同时登记所有 stage architecture 到 `_ARCH_TO_MODEL_TYPE`，注册统一 `NemotronVoiceChatConfig`，让 alias、path-basename fallback 与 `default_deploy_config_name` 指向同一 pipeline；不得只依赖路径猜测。 -- 禁止：只注册 thinker 或只提供一个 alias；让空的 `model_arch` 覆盖 checkpoint architecture；让 sync 与 async deploy 配置解析出不同的 stage 拓扑。 -- 验收：用 bare `config.json` 和三个 architecture 分别验证 model type、pipeline、stage config 与默认 deploy 名称；sync/async 配置只切换 processor wiring，并保持三 stage 顺序与 stage architecture 一致。 -^[PR #5842] - -### CONF-5e — MiniCPM-o 部署 profile 必须一致传递 CFM Graph 开关 - -- 触发：修改 MiniCPM-o 4.5 bundled deploy YAML 或 Code2Wav connector extra 中的 CFM CUDA Graph 开关、缓存上限或配置传递。\n- 强制：`minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml`、`minicpmo_4_5_3gpu.yaml` 和 `minicpmo_4_5_8x4090.yaml` 必须一致声明 `enable_cfm_graph` 与 `cfm_max_graphs`；当前默认值为 `true` 与 `32`，并传入 `MiniCPMO45Code2Wav` 的 `_cfm_graph_config` 和 `BatchedToken2Wav`。非 CUDA 设备仍必须回退 eager。\n- 禁止：只更新一个 deploy profile、依赖未记录的默认值，或把配置存在误认为 graph 已在非 CUDA 环境生效；不得把模型专有开关扩展成所有模型的通用配置合同。\n- 验收：解析四份 deploy 配置并断言开关和上限一致，追踪非默认值到 `_cfm_graph_config` 与 `BatchedToken2Wav`；覆盖开关关闭和非 CUDA fallback，并确认 stage topology 未发生额外变化。\n^[PR #6082] - -### CONF-5f — π0 的 checkpoint、registry 与 deploy 必须闭合为单 diffusion stage - -- 触发：新增或修改 π0 的 `pipeline_registry.py`、`vllm_omni/deploy/pi0.yaml`、`OmniDiffusionConfig` 的 `type: pi0` autodetect，或 `Pi0Pipeline` 的 stage declaration。 -- 强制：让 `pipeline: pi0`、`OMNI_PIPELINES["pi0"]`、`Pi0Pipeline` 与 checkpoint `type: pi0` resolver 闭合；deploy 配置只能声明一个 diffusion stage，最终输出类型为 `action`，并让 `policy_server_config` 的 `image_resolution`、`action_horizon`、`action_dim`、camera 数量与 `model_config` 保持一致。 -- 禁止：只凭 checkpoint 路径或文件存在推断 pipeline，额外添加 model-executor 层或多 stage topology，或让 OpenPI metadata 与最终 stage/model config 的动作尺寸不一致。 -- 验收：无显式 deploy 配置的 checkpoint autodetect 能解析到 `Pi0Pipeline`；解析最终逐 stage config 断言只有一个 diffusion stage、`final_output_type=action`，并核对 `pi0.yaml` 与 websocket handshake metadata 的关键字段一致。^[PR #4222] - -### CONF-5g — MiniCPM-o NPU Code2Wav 开关必须按 Stage 2 传递 - -- 触发：修改 MiniCPM-o 4.5 bundled deploy YAML，或新增/调整 `code2wav_enable_npu_graph`、`code2wav_max_npu_graphs` 这类 Ascend Code2Wav graph 配置。 -- 强制：`minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml` 和 `minicpmo_4_5_3gpu.yaml` 必须在 `platforms.npu.stages` 中按 stage 声明配置；Stage 0/1 保持 `compilation_config.cudagraph_mode: PIECEWISE`，Stage 2 只通过 `additional_config` 传递 graph 开关和上限，默认值为 `true` 与 `32`，并保持最终 runner 的 outer eager 语义。所有断言必须基于 `_apply_platform_overrides` 与 `merge_pipeline_deploy` 展开的最终 stage config。 -- 禁止：把 Code2Wav NPU 开关放入 shared connector extra、全局配置或其他 stage；只修改一个 profile；以 YAML 原文或字段存在推断配置已到达 Code2Wav consumer；用全局 eager override 破坏 Stage 0/1 的 `PIECEWISE` 配置。 -- 验收：逐一解析三个 profile，断言 connector extra 不含两个 Code2Wav key，NPU 展开结果包含 Stage 0/1 的 `PIECEWISE`、Stage 2 的 `enforce_eager: true`、`code2wav_enable_npu_graph: true` 和 `code2wav_max_npu_graphs: 32`，并覆盖开关关闭和缓存上限为 0 的路径。^[PR #5604] - ### CONF-6a — MOSS-TTS-Local 部署必须采用官方采样参数 - 触发：修改 MOSS-TTS-Local 的 deploy YAML 或其默认采样参数时。 From 66afc2d5c1181cad70e38c93d0d4043ed4786f9c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:50:30 +0800 Subject: [PATCH 195/465] knowledge(vllm-omni): split the request input contract out of serving rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serving/rules.md had 2867 bytes of headroom against 62 remaining units routing to it, the highest pressure of any page in the tree. Same pre-emptive split as the configuration one alongside it, landed at a unit boundary rather than after a halt. 请求输入合同 was already a named section holding one subject, so it moves whole and becomes the page rather than being cut to fit a byte budget: SERV-4a-4h, SERV-4l-4m -> rules-request-input.md public request field validation, source conflicts, alias and extras normalisation, consumer view Rule IDs unchanged, registered once in _index.md, and five group rows retargeted. Four of those name SERV-4c or SERV-4d from other groups -- core, chat-multimodal- contract, endpoint-capability -- which is the case the diffusion split got wrong by claiming a whole ID range for one page: a rule can be cited by a group that does not own it, and every citation has to move, not just the owning row. The fifth row is batch-chat-contract, whose SERV-4i-4k were never on this page at all; they have lived in rules-batch-chat.md since an earlier split and the row still pointed here. One inbound anchor broke -- models/lingbot-video/rules.md links at serving/rules.md#serv-4c-... -- and is retargeted. Sixth split, sixth breakage, second one found by check_anchors.py rather than by a reviewer or an ad-hoc grep. No unit is consumed and audited_sha is untouched. Gates: tree 0 errors / 14 reminders, wiki lint 0 errors across 337 pages, release audit CLEAN, 107 anchored links clean. Headroom after: rules.md 9902 B. That clears the last page split_risk.py was flagging with high unit pressure, so the driver can resume at unit 185 without a scheduled split ahead of it. Co-Authored-By: Claude Opus 5 (1M context) --- .../vllm-omni/components/serving/_index.md | 3 +- .../components/serving/rules-request-input.md | 112 ++++++++++++++++++ .../vllm-omni/components/serving/rules.md | 110 +---------------- .../vllm-omni/models/lingbot-video/rules.md | 2 +- 4 files changed, 120 insertions(+), 107 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-request-input.md diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index edbca49a6..10fece8ce 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -1,7 +1,7 @@ --- title: "Serving" created: 2026-07-10 -updated: 2026-09-03 +updated: 2026-09-04 type: index tags: [vllm-omni, components, serving] sources: [] @@ -33,3 +33,4 @@ sources: [] | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | | engine startup/shutdown 顺序、stage 生命周期、full-duplex 与 CFG companion | [engine 生命周期规则](rules-engine-lifecycle.md) | +| 公开请求字段的校验、来源冲突、alias/extras 归一与 consumer view | [请求输入合同](rules-request-input.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md new file mode 100644 index 000000000..fab682e59 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -0,0 +1,112 @@ +--- +title: "请求输入合同" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #3805", "PR #5374", vllm_omni/entrypoints/openai/, vllm_omni/inputs/] +confidence: high +--- + +# 请求输入合同 + +`SERV-4a`–`SERV-4m`：公开请求字段的校验、限界与 owner。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口。 + +## SERV-4a — 公开字段由 serving 显式拥有 + +- 触发：修改请求 allowlist、冲突字段集或兼容输入。 +- 强制：逐项绑定真实 consumer，公开字段由 serving 边界显式声明。 +- 禁止：从包含 tensor、KV 状态或运行时中间量的内部结构反射生成公开字段。 +- 验收：加入一个内部同名字段反例，证明它不会被误算成公开 root 字段。 + +## SERV-4b — 多来源输入验证前不得合并 + +- 触发：请求同时支持 flattened、raw nested、声明字段、alias 或 canonical container。 +- 强制：保留来源直到冲突检查结束；验证通过后若用字典展开构造并集，必须注明各映射 + 已经不相交。 +- 禁止：用 `or`、字典展开或 `update()` 决定重复值，制造未声明优先级。 +- 验收：重复字段返回明确 4xx，不重叠字段全部到达最终 consumer。 + +## SERV-4c — 入口接受必须闭环到每个生产消费者 + +- 触发：新增请求字段或改变字段分流。 +- 强制：对每条 dispatcher 追踪字段到 engine、pipeline、prompt 或 sampling 参数。 +- 禁止：用 helper 返回值或 HTTP 成功代替传播证明。 +- 验收：真实请求对象同时覆盖默认值和非默认值，并断言最终 consumer。 + +## SERV-4d — 同一请求合同错误跨 dispatcher 保持同一响应合同 + +- 触发：同一非法输入可进入 diffusion-only、multi-stage 或其他多个 dispatcher。 +- 强制：使用一致的 status、错误类型和消息策略，并在公共边界转换一次。 +- 禁止：一路本地映射为 4xx，另一路交给远处通用 `ValueError` 捕获。 +- 验收：同一冲突输入经过每条受影响 dispatcher 时响应等价，且都在 engine/pipeline + 调用前失败。 + +## SERV-4e — 请求期弃用信号必须对 operator 可见 + +- 触发：serving 路径继续接收 deprecated 输入。 +- 强制：使用项目 logger 的 `warning_once` 或明确限频策略。 +- 禁止：使用仅写 stderr 且按调用点过滤的 `warnings.warn`。 +- 验收：合法旧输入恰好记录一次警告；因冲突返回 4xx 的输入不记录兼容警告；用户响应 + 合同与日志合同分别断言。 + +## SERV-4f — Serving 只编译当前 slice 拥有的请求语义 + +- 触发：同一 serving 字段存在 flattened、nested、canonical 或 legacy 来源。 +- 强制：在 request mutation、preprocess 和 dispatcher 分支之前完成一次来源校验，并 + 产出当前 slice 限定字段的 consumer view。 +- 禁止：dispatcher 重读 raw request 或重新决定优先级；为了 request-extra + normalization 把 topology、模型能力、逐 stage 参数或其他 owner 吸进完整 compiler。 +- 验收：root control + nested extras 分别经过 pure/mixed dispatcher 到达 prompt、 + AR metadata 与 diffusion sampling consumer；registry 字段与 service control 重名时 + 仍只有一个 owner。 + +## SERV-4g — 多来源合同编码前必须完成来源矩阵 + +- 触发：一个语义存在多个来源、dispatcher 或 stage scope。 +- 强制：按 [source-consumer decision matrix](../../../../general/review/guides/review-execution-contract.md#source-consumer-decision-matrix) + 标明路由、重复拒绝、不适用和 defaults；兼容写法默认进入同一 consumer scope，只有 + 矩阵声明不同语义时才能分流。规范化结果拆成多个 consumer view 时默认互不重叠， + 同一字段确需进入多个 view 时逐一命名最终 consumer。 +- 禁止：矩阵缺失时声称实现或审查完成；由字段集合运算、输入写法或 dispatcher 末端 + defaults 隐式决定 scope。 +- 验收：每个来源组合都有明确 decision 和生产路径证据；每个 consumer view 的重叠项 + 都有显式最终 consumer，不存在接受后丢弃或末端重新读取 raw request。 + +## SERV-4h — 请求合同膨胀时停止逐评论修补 + +- 触发：生产 diff 超过预算上限 1.5 倍、出现第二个重叠语义 owner，或下一审查波次 + 再次发现同一 owner 漏洞。 +- 强制：执行 [架构重置验收](../../../../general/review/guides/code-taste.md#架构重置怎样验收)， + 重新确认唯一最终产物、删除清单和规模上限。 +- 禁止：继续堆 helper、compatibility branch 或 reviewer-specific patch。 +- 验收：恢复编码前 owner、consumer、删除项和 diff 预算都有可检查记录。 + +## SERV-4l — request-level LoRA 必须贯穿每种 stage-0 submission 形态 + +- 触发：修改 chat adapter 解析、`AsyncOmni.generate`、ordinary/streaming request submission、 + stage-0 input processing 或 LoRA request 类型。 +- 强制：OpenAI chat `_maybe_get_adapters()` 得到的同一个 request-level `LoRARequest` 必须传入 + `AsyncOmni.generate`，再贯穿普通 `add_request_async`、streaming 首 chunk、每个 + `add_streaming_update_async` 与无 chunk/final marker，最终由 + `AsyncOmniEngine._build_add_request_message` 传给**非 diffusion stage 0** 的 vLLM input + processor。`None` 同样原样传播，不能复用上一请求 adapter。 +- 禁止：只修非流式提交；把单个 request-level LoRA 广播成每 stage adapter；绕过既有 + `resolve_sampling_params_list` 的 stage cardinality 校验；以 adapter 已加载证明 request 已选择 + 它。该参数在 target 的 engine APIs 仍标作 `Any`，不是类型安全合同；评审明确要求收窄类型， + 后续应统一为 `LoRARequest | None` 而非继续扩散 `Any`。 +- 验收：用 identity sentinel 分别覆盖普通 prompt、streaming 有 chunk、streaming 空输入/final + marker 与 `None`；单 stage sampling 参数包成长度 1，多 stage 参数长度必须等于 stage 数。 + 断言 stage-0 input processor 收到同一对象，并另外验证 downstream stage 的 LoRA 仍由对应 + `sampling_params_list[i].lora_request` 拥有，默认参数对象不被污染。GPU 验收必须先确认 adapter + ID 已加载，再证明 deterministic token/logprob 与 base 不同；#5369 另报的 `AsyncOmni.add_lora` + control-RPC 反序列化成 list 问题不在本修复范围。^[PR #5374] + +## SERV-4m — API server 必须规范化 CLI 参数并同步 positional model + +- 触发：修改 `vllm_omni.entrypoints.openai.api_server` 的 CLI parser、模型参数来源或直接运行入口。 +- 强制：直接运行 API server 时复用 upstream `FlexibleArgumentParser`，使下划线参数与连字符参数保持一致的规范化行为；解析后若 `args.model_tag` 非空，必须将 `args.model` 同步为该值，以确保 positional model 不会退回 vLLM `ModelConfig` 的默认模型。 +- 禁止：使用不支持 upstream flag normalization 的独立 argparse parser；仅在 `args.model` 缺失时反向填充 `model_tag`，导致 positional model 仍未传入实际 model consumer；把 `--omni` 的兼容占位参数解释为改变直接入口运行模式的开关。 +- 验收：分别验证 positional model、`--model`、`--gpu_memory_utilization` 等下划线参数和连字符参数均可解析，并断言 `args.model` 与 `args.model_tag` 一致且服务加载请求模型，不会回退到默认 LLM。^[PR #3805] + +engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)；故障隔离见 [fault isolation 规则](rules-fault-isolation.md)。 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 498846ec3..08580a37c 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -35,17 +35,17 @@ confidence: high | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| -| `core` | 每次 serving 审查 | `SERV-4c` | +| `core` | 每次 serving 审查 | `SERV-4c`，见 [请求输入合同](rules-request-input.md) | | `streaming-format` | SSE、audio streaming、format/default/capability | `SERV-1a`, `SERV-1b` | | `media-ingress` | video reference、decoder registry/backend、mixed capability、bounded upload/conversion | `SERV-1c`–`1e` | | `metrics-lifecycle` | metrics、gauge、replica、collector | `SERV-2a`, `SERV-2b` | | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | -| `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c` + 命中模型规则 | -| `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d`, `SERV-5d` | +| `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | +| `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | | `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection、replica fault isolation | `SERV-5a`–`SERV-5e`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | -| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`, `SERV-4b`, `SERV-4c`, `SERV-4d`, `SERV-4e`, `SERV-4f`, `SERV-4g`, `SERV-4h`, `SERV-4l` | -| `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`, `SERV-4j`, `SERV-4k` | +| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`–`4h`, `SERV-4l`–`4m`，全部见 [请求输入合同](rules-request-input.md) | +| `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | ## SERV-1a — 所有可预判错误在第一个 streaming chunk 前返回 @@ -205,103 +205,3 @@ confidence: high locator 建无界共享 side table；把完整 base64/路径写日志。 - 验收：同长度覆盖、并发改写、named+inline、speaker cache 和 prefix hit 均不复用旧内容； 非法路径不 stat，日志只含截断值且命中真实 logger hierarchy。 ^[PR #5670] - -## 请求输入合同 - -### SERV-4a — 公开字段由 serving 显式拥有 - -- 触发：修改请求 allowlist、冲突字段集或兼容输入。 -- 强制：逐项绑定真实 consumer，公开字段由 serving 边界显式声明。 -- 禁止：从包含 tensor、KV 状态或运行时中间量的内部结构反射生成公开字段。 -- 验收：加入一个内部同名字段反例，证明它不会被误算成公开 root 字段。 - -### SERV-4b — 多来源输入验证前不得合并 - -- 触发：请求同时支持 flattened、raw nested、声明字段、alias 或 canonical container。 -- 强制：保留来源直到冲突检查结束；验证通过后若用字典展开构造并集，必须注明各映射 - 已经不相交。 -- 禁止：用 `or`、字典展开或 `update()` 决定重复值，制造未声明优先级。 -- 验收：重复字段返回明确 4xx，不重叠字段全部到达最终 consumer。 - -### SERV-4c — 入口接受必须闭环到每个生产消费者 - -- 触发：新增请求字段或改变字段分流。 -- 强制：对每条 dispatcher 追踪字段到 engine、pipeline、prompt 或 sampling 参数。 -- 禁止：用 helper 返回值或 HTTP 成功代替传播证明。 -- 验收：真实请求对象同时覆盖默认值和非默认值，并断言最终 consumer。 - -### SERV-4d — 同一请求合同错误跨 dispatcher 保持同一响应合同 - -- 触发：同一非法输入可进入 diffusion-only、multi-stage 或其他多个 dispatcher。 -- 强制：使用一致的 status、错误类型和消息策略，并在公共边界转换一次。 -- 禁止：一路本地映射为 4xx，另一路交给远处通用 `ValueError` 捕获。 -- 验收：同一冲突输入经过每条受影响 dispatcher 时响应等价，且都在 engine/pipeline - 调用前失败。 - -### SERV-4e — 请求期弃用信号必须对 operator 可见 - -- 触发：serving 路径继续接收 deprecated 输入。 -- 强制：使用项目 logger 的 `warning_once` 或明确限频策略。 -- 禁止：使用仅写 stderr 且按调用点过滤的 `warnings.warn`。 -- 验收：合法旧输入恰好记录一次警告；因冲突返回 4xx 的输入不记录兼容警告；用户响应 - 合同与日志合同分别断言。 - -### SERV-4f — Serving 只编译当前 slice 拥有的请求语义 - -- 触发：同一 serving 字段存在 flattened、nested、canonical 或 legacy 来源。 -- 强制：在 request mutation、preprocess 和 dispatcher 分支之前完成一次来源校验，并 - 产出当前 slice 限定字段的 consumer view。 -- 禁止：dispatcher 重读 raw request 或重新决定优先级；为了 request-extra - normalization 把 topology、模型能力、逐 stage 参数或其他 owner 吸进完整 compiler。 -- 验收：root control + nested extras 分别经过 pure/mixed dispatcher 到达 prompt、 - AR metadata 与 diffusion sampling consumer；registry 字段与 service control 重名时 - 仍只有一个 owner。 - -### SERV-4g — 多来源合同编码前必须完成来源矩阵 - -- 触发：一个语义存在多个来源、dispatcher 或 stage scope。 -- 强制：按 [source-consumer decision matrix](../../../../general/review/guides/review-execution-contract.md#source-consumer-decision-matrix) - 标明路由、重复拒绝、不适用和 defaults；兼容写法默认进入同一 consumer scope，只有 - 矩阵声明不同语义时才能分流。规范化结果拆成多个 consumer view 时默认互不重叠， - 同一字段确需进入多个 view 时逐一命名最终 consumer。 -- 禁止：矩阵缺失时声称实现或审查完成；由字段集合运算、输入写法或 dispatcher 末端 - defaults 隐式决定 scope。 -- 验收：每个来源组合都有明确 decision 和生产路径证据；每个 consumer view 的重叠项 - 都有显式最终 consumer，不存在接受后丢弃或末端重新读取 raw request。 - -### SERV-4h — 请求合同膨胀时停止逐评论修补 - -- 触发：生产 diff 超过预算上限 1.5 倍、出现第二个重叠语义 owner，或下一审查波次 - 再次发现同一 owner 漏洞。 -- 强制：执行 [架构重置验收](../../../../general/review/guides/code-taste.md#架构重置怎样验收)， - 重新确认唯一最终产物、删除清单和规模上限。 -- 禁止：继续堆 helper、compatibility branch 或 reviewer-specific patch。 -- 验收：恢复编码前 owner、consumer、删除项和 diff 预算都有可检查记录。 - -### SERV-4l — request-level LoRA 必须贯穿每种 stage-0 submission 形态 - -- 触发：修改 chat adapter 解析、`AsyncOmni.generate`、ordinary/streaming request submission、 - stage-0 input processing 或 LoRA request 类型。 -- 强制：OpenAI chat `_maybe_get_adapters()` 得到的同一个 request-level `LoRARequest` 必须传入 - `AsyncOmni.generate`，再贯穿普通 `add_request_async`、streaming 首 chunk、每个 - `add_streaming_update_async` 与无 chunk/final marker，最终由 - `AsyncOmniEngine._build_add_request_message` 传给**非 diffusion stage 0** 的 vLLM input - processor。`None` 同样原样传播，不能复用上一请求 adapter。 -- 禁止：只修非流式提交；把单个 request-level LoRA 广播成每 stage adapter；绕过既有 - `resolve_sampling_params_list` 的 stage cardinality 校验；以 adapter 已加载证明 request 已选择 - 它。该参数在 target 的 engine APIs 仍标作 `Any`，不是类型安全合同；评审明确要求收窄类型， - 后续应统一为 `LoRARequest | None` 而非继续扩散 `Any`。 -- 验收：用 identity sentinel 分别覆盖普通 prompt、streaming 有 chunk、streaming 空输入/final - marker 与 `None`；单 stage sampling 参数包成长度 1，多 stage 参数长度必须等于 stage 数。 - 断言 stage-0 input processor 收到同一对象，并另外验证 downstream stage 的 LoRA 仍由对应 - `sampling_params_list[i].lora_request` 拥有，默认参数对象不被污染。GPU 验收必须先确认 adapter - ID 已加载，再证明 deterministic token/logprob 与 base 不同；#5369 另报的 `AsyncOmni.add_lora` - control-RPC 反序列化成 list 问题不在本修复范围。^[PR #5374] - -### SERV-4m — API server 必须规范化 CLI 参数并同步 positional model - -- 触发：修改 `vllm_omni.entrypoints.openai.api_server` 的 CLI parser、模型参数来源或直接运行入口。 -- 强制：直接运行 API server 时复用 upstream `FlexibleArgumentParser`，使下划线参数与连字符参数保持一致的规范化行为；解析后若 `args.model_tag` 非空，必须将 `args.model` 同步为该值，以确保 positional model 不会退回 vLLM `ModelConfig` 的默认模型。 -- 禁止：使用不支持 upstream flag normalization 的独立 argparse parser；仅在 `args.model` 缺失时反向填充 `model_tag`，导致 positional model 仍未传入实际 model consumer；把 `--omni` 的兼容占位参数解释为改变直接入口运行模式的开关。 -- 验收：分别验证 positional model、`--model`、`--gpu_memory_utilization` 等下划线参数和连字符参数均可解析，并断言 `args.model` 与 `args.model_tag` 一致且服务加载请求模型，不会回退到默认 LLM。^[PR #3805] - diff --git a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md index 224fc2629..f07d68b8d 100644 --- a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md +++ b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md @@ -10,7 +10,7 @@ sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/d # LingBot-Video 规则 只有 `LBV-数字字母` 是可审计规则 ID。共享请求传播与错误一致性先应用 -[SERV-4c/4d](../../components/serving/rules.md#serv-4c-入口接受必须闭环到每个生产消费者)， +[SERV-4c/4d](../../components/serving/rules-request-input.md#serv-4c-入口接受必须闭环到每个生产消费者)， 本页只写 LingBot 专有语义。 ## Direct 代码快速入口 From b447cd630e48e81158cefb08306c15f1723c5851 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:51:44 +0800 Subject: [PATCH 196/465] knowledge(vllm-omni): sync 199ef777 (PR #6276) Upstream-Commit: 199ef7779c1b7650a0e9df0951e0a2b9da4e0190 Upstream-PR: #6276 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b3c0b84fd..3eee395af 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d7216ef5144866c2f039947e1cc396b1984fcf09 + previous_audited_sha: 3c6648d8a54902297ca7d46815e9883c1de6b204 audited_ref: main - audited_sha: 3c6648d8a54902297ca7d46815e9883c1de6b204 + audited_sha: 199ef7779c1b7650a0e9df0951e0a2b9da4e0190 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 35a03bb60..72b1cf733 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3c6648d8a54902297ca7d46815e9883c1de6b204`. The canonical machine baseline +199ef7779c1b7650a0e9df0951e0a2b9da4e0190`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ed375db78..770d358d9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3c6648d8` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 199ef777` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ff37a0069..cf2d47b4a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3c6648d8` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 199ef777` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 917dbe7a7..9153d3228 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3c6648d8` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 199ef777` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index bb1a1d5bd..2c61d683b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3c6648d8` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 199ef777` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 671fea92c..de00b0303 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3c6648d8` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 199ef777` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 10fece8ce..2e17e760f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3c6648d8` 验证存在 +- 源码校验：以上路径均已在 `main @ 199ef777` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index fdf6741e5..efbb25620 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3c6648d8`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 199ef777`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `d7216ef5` 本轮将配置包的延迟导出与 NPU 模型补丁时序写入现有组件规则；测试文件归 CI/testing owner，未写入规则页。不能据此推断所有平台或所有模型都存在相同的导入循环，或仅凭测试通过就证明完整 NPU 启动链路已覆盖。 +本次相对 `3c6648d8` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from d7216ef5 \ - --to 3c6648d8 \ + --from 3c6648d8 \ + --to 199ef777 \ --repo \ --mode report-only ``` From 860d22c63412c6bd3fb1f78dfb9ce9fa0c1a812f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:55:34 +0800 Subject: [PATCH 197/465] knowledge(vllm-omni): sync b283215c (PR #6186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b283215c28d56d24b3f72a120b8f34222699aeda Upstream-PR: #6186 Knowledge-Delta: 沉淀 MiniMax Music 3 的多组件部署闭合、CFG pair 调度、companion 原子 admission 与 speech adapter 请求合同。 --- adapters/vllm_omni/release_baseline.yaml | 16 ++++++++-------- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../configuration/rules-topology-profiles.md | 10 +++++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-bridge-batch.md | 9 ++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 16 +++++++++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 12 files changed, 51 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3eee395af..32221adaa 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3c6648d8a54902297ca7d46815e9883c1de6b204 + previous_audited_sha: 199ef7779c1b7650a0e9df0951e0a2b9da4e0190 audited_ref: main - audited_sha: 199ef7779c1b7650a0e9df0951e0a2b9da4e0190 + audited_sha: b283215c28d56d24b3f72a120b8f34222699aeda # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 86 - sha256: 039140fb0a8e95f13015d11af550db8ed16e6071448889f243c1525de5dff493 + count: 88 + sha256: 96d973728f1d67a959121c4aa0d8ba86cff32043ec6f90458f48d1d766acb8af diffusion: count: 65 sha256: b0a814ade12af087b87d45e0a0583cf844456851b3dc052496bb996823b4bbb4 pipelines: - count: 58 - sha256: dcec9759334dd9aff66464720aaded39c886dff9d84ec2e0ca6950a454db2df6 + count: 59 + sha256: c2516b9fb6124aa0774a05390a6f05c08ed6998e287b03c8252d85338934933e deploy_yamls: - count: 86 - sha256: f766ce25bee9d46abccfe6a245ac51af99d7733c5be72b2d29032e0bbe86840f + count: 88 + sha256: d4a1840627fced8bac9b089fe8b13b69bd39ebee5264fd465d42f6c389d094e5 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 72b1cf733..bb9cf0fc5 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -199ef7779c1b7650a0e9df0951e0a2b9da4e0190`. The canonical machine baseline +b283215c28d56d24b3f72a120b8f34222699aeda`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 770d358d9..12e6fa1fa 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 199ef777` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b283215c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md index 8eaa7289e..e5717cc81 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #6082", vllm_omni/deploy/] +sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #6082", vllm_omni/deploy/, "PR #6186"] confidence: high --- @@ -69,3 +69,11 @@ confidence: high - 验收：逐一解析三个 profile，断言 connector extra 不含两个 Code2Wav key，NPU 展开结果包含 Stage 0/1 的 `PIECEWISE`、Stage 2 的 `enforce_eager: true`、`code2wav_enable_npu_graph: true` 和 `code2wav_max_npu_graphs: 32`，并覆盖开关关闭和缓存上限为 0 的路径。^[PR #5604] 部署 YAML 的展开顺序见 [deploy-yaml](deploy-yaml.md)；stage 执行合同见 [model-executor 规则](../model-executor/rules.md)。 + +## CONF-5h — 多组件 checkpoint 必须闭合 root 身份、stage 子目录与默认部署 + +- 触发：新增由多个子目录组成、root `config.json` 自报 composite architecture，或需要按 stage 加载不同 checkpoint/tokenizer 子目录的模型，并同步新增 pipeline、默认 deploy 或架构注册。 +- 强制：闭合 checkpoint `model_type`、pipeline registry key、`model_arch`、`default_deploy_config_name` 与 registry architecture；每个 stage 明确 `model_subdir`/`tokenizer_subdir` 和实际权重根目录，最终 stage config 保留 CFG 等 engine gate 所需声明。 +- 禁止：只依赖路径 basename、文件存在或 `trust_remote_code` 推断 pipeline；让 root architecture 覆盖 stage architecture；让单卡与多卡 profile 复制不同 topology 或行为合同。 +- 验收：用 bare root config、无显式 deploy 的 startup 和 structured/legacy 路径断言 pipeline、默认 deploy、stage 子目录、architecture 与 sampling-extra key 均进入最终逐 stage config；单卡与双卡配置除设备 placement 外保持一致。 ^[PR #6186] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index cf2d47b4a..f34d181ac 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 199ef777` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b283215c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9153d3228..a98bab4bb 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 199ef777` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b283215c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2c61d683b..b243583cd 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 199ef777` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b283215c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md index bca50c556..d31e0cf3c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666", "PR #5491"] +sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666", "PR #5491", "PR #6186"] confidence: high --- @@ -132,3 +132,10 @@ Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x - 禁止：用内部 engine request ID 冒充协议 request ID；在模型输出验证前确认事件已应用；原地重试失败 chunk；让通用 session 层解析 LingBot camera/action schema；在没有 session-affine routing 时启用多副本 AR stage；把内部 tick contract 当作公共 HTTP/WebSocket 协议。 - 验收：覆盖乱序/重复/过期事件、队列背压、交错 `A0 -> B0 -> A1`、ID 分离、缺失或错误元数据、reducer commit 失败、失败后 reset、cleanup 失败 tombstone 与 close retry；同时验证 consumer 对非单副本 stage fail closed，并断言普通 `AsyncOmni` 输出 ID 合同不变。^[PR #5491] +## EXEC-1m — CFG 成对请求必须按 identity 保持 batch 行步进一致 + +- 触发：模型使用 classifier-free guidance，以 `cond`/`uncond` 两个 engine row 解码，或在 batch compaction、prefill/decode 与 request reorder 下维护成对状态。 +- 强制：通过 `cfg_pair_id` 或注册的 companion suffix 从 external/user request identity 建立 pair；在 Scheduler 构造前安装通用 pairing patch；不完整 pair 必须等待，完整 pair 保持相邻并在每次 schedule 后 equalize progress、一起结束，同时按 pair/request id 保存状态而不是按 batch row 保存。 +- 禁止：用 row adjacency 或 internal UUID 推断 pair；让 prefix-cache 不对称命中、chunked prefill 或单边 preemption 使两行错位；只结束一方、把 per-request seed/position metadata 广播给整批，或让非 CFG stage 被无条件改写。 +- 验收：覆盖 suffix/显式 pair id、companion 缺失与到达、row reorder/compaction、chunked prefill、prefix-cache control、一起 finish、非 CFG no-op 和 patch 安装时序；同批不同 seed 与位置必须保持各自归属。 ^[PR #6186] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index de00b0303..815d67777 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 199ef777` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b283215c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2e17e760f..c21ee1fc4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 199ef777` 验证存在 +- 源码校验：以上路径均已在 `main @ b283215c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 923b29576..2ae56f132 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186"] confidence: high --- @@ -114,6 +114,13 @@ confidence: high - 禁止：把新的 model-type 分支或采样 mutation 加回 `serving_speech.py`；让 adapter 直接污染 engine 的共享默认 sampling 参数；保留没有 adapter 支撑的 legacy detector，或用只验证 dispatch 到达的测试替代最终 `SamplingParams` 结果断言。 - 验收：覆盖 adapter registry/detection 无歧义、`extra_params -> apply_sampling_overrides -> seed` 顺序、无与有 `max_new_tokens`、CosyVoice3/GLM-TTS 动态边界、Ming stop token、Audex request-id/CFG/TTA 以及共享默认参数不变；测试必须检查传给 engine 的实际 sampling 参数。 ^[PR #5272] +### SERV-5j — TTS adapter 必须承载模型专属请求合同与采样覆盖 + +- 触发：speech endpoint 接入没有 speaker/reference-audio 的 text-to-music 模型，或模型的必填字段、采样控制和普通 TTS 合同不同。 +- 强制：注册 adapter 并由 adapter metadata 完成 detection；adapter 负责 `validate()`、prompt/token-id 构造和 `apply_sampling_overrides()`，明确 `input` lyrics、`instructions` caption、固定 sampling、长度上限及被拒绝的 TTS 字段，shared serving 只负责公共 dispatch。 +- 禁止：在 `serving_speech.py` 增加 model-type 分支；静默忽略 `voice`、reference audio、temperature 或 `stream` 等不支持字段；把无 speaker 模型套用普通 TTS 的 voice 或 temperature 语义。 +- 验收：通过真实 adapter detection 和 `/v1/audio/speech` 请求覆盖缺失/空 lyrics、缺失/空 caption、unsupported fields、长度边界、tokenizer 校验与非流式输出，并断言送入 engine 的实际 prompt、sampling 参数和 model type。 ^[PR #6186] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 @@ -169,6 +176,13 @@ confidence: high - 禁止：用 `fatal=True` 将单请求输入错误升级为 engine-wide failure；cleanup 已移除 request state 后仍返回成功或继续 duplex bookkeeping；让一个 request 的 prewarm 失败拖垮同批其他请求。 - 验收：覆盖缺失 `prompt_token_ids`、downstream submit 失败和 duplex submit fall-through，断言错误只归属目标 request、engine/thread 仍存活、下游未被错误预热，且无 fence、时间戳、counter 或 session 残留。^[PR #6033] +### SERV-6f — 必需 CFG companion 的构造与 admission 必须原子完成 + +- 触发：engine 为 CFG 请求展开 companion，且 guidance 是模型必需能力，或 companion 需要经过 input processor 才能获得完整 Omni request metadata。 +- 强制：先构造并处理全部 companions，再把 parent 和 companions 一起入队；处理后用 `upgrade_to_omni_request` 恢复 `additional_information`/`global_request_id` 等 Omni 字段；任一构造或处理失败都向调用方传播并保持队列无变化，非 CFG expansion 返回空列表并保留普通 admission。 +- 禁止：先 enqueue parent 再吞掉 companion 异常；只入队一侧、让 companion 丢失 external/global request identity，或在 Scheduler 建立后才安装 CFG pairing gate。 +- 验收：分别模拟 prompt expansion、input processing 和 tokenizer 失败，断言 parent/companion 均未入队；成功路径断言两者 metadata、ID suffix、sampling params 和 pair identity 完整；无 companion 的普通请求仍正常 admission。 ^[PR #6186] + ### SERV-8a — 本地 stage launch 必须临时应用 runtime.env - 触发：修改 `StageRuntime` 的本地 LLM/diffusion replica launch、`runtime_cfg.env`、stage device scope 或 launch lock。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index efbb25620..2569b8337 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 199ef777`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b283215c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `3c6648d8` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `199ef777` PR #6186 新增 `MiniMaxAI/MiniMax-Music3`。本轮仅沉淀多组件部署闭合、CFG 成对调度、companion 原子 admission 和 speech adapter 请求合同；不能据此推断音质、并发性能或端到端 bit-identical，模型专属 prompt/window/weights 细节未新增规则。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 3c6648d8 \ - --to 199ef777 \ + --from 199ef777 \ + --to b283215c \ --repo \ --mode report-only ``` From abc20e5d9a2e87d690f03bd1d3a6b6b17ffaf66a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 06:59:40 +0800 Subject: [PATCH 198/465] knowledge(vllm-omni): sync 0a1846aa (PR #6094) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 0a1846aa458283a13c1a7b5a96ab585937f9726c Upstream-PR: #6094 Knowledge-Delta: 新增 paged_scheduler 的配置边界、native KV 控制面原子 CFG 分配与生命周期清理，以及 HunyuanImage3 的 paged attention 注册和最大图输入 profile 合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-output-lifecycle.md | 2 +- .../components/diffusion/rules-system-runtime.md | 7 +++++++ .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../repos/vllm-omni/models/hunyuan-image3/rules.md | 11 +++++++++-- 13 files changed, 38 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 32221adaa..9ceacb82e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 199ef7779c1b7650a0e9df0951e0a2b9da4e0190 + previous_audited_sha: b283215c28d56d24b3f72a120b8f34222699aeda audited_ref: main - audited_sha: b283215c28d56d24b3f72a120b8f34222699aeda + audited_sha: 0a1846aa458283a13c1a7b5a96ab585937f9726c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index bb9cf0fc5..22313a492 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b283215c28d56d24b3f72a120b8f34222699aeda`. The canonical machine baseline +0a1846aa458283a13c1a7b5a96ab585937f9726c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 12e6fa1fa..bc21c3cac 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b283215c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0a1846aa` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 7c2d0b087..523a290e5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py"] --- # vLLM-Omni 配置开发门禁 @@ -117,6 +117,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR 决定 KV 大小。 - 验收：记录 pin、并发、长度与峰值，真实 decode 路径完成；任何吞吐交换都单独测量。 +### CONF-2b — paged_scheduler 的 KV sizing 字段必须归一化并保持 dense_legacy 惰性 + +- 触发：为 diffusion stage 增加或修改 `kv_cache_memory_bytes`、`gpu_memory_utilization`、`max_num_batched_tokens`、`max_model_len`，或改变 `dense_legacy` 与 `paged_scheduler` 的配置投影。 +- 强制：这些字段必须经同一 diffusion owner 归一化；只有 `paged_scheduler` 才能传入 native `VllmConfig` 的 cache/scheduler 配置。正值 `kv_cache_memory_bytes` 固定 KV 预算，`None` 或 `0` 使用自动 sizing；`max_model_len=-1` 必须保留 native auto-fit sentinel，同时为 Scheduler 解析出正的 admission bound。`kv_cache_memory_bytes` 不得为负，`gpu_memory_utilization` 必须在 `(0,1]`，`max_num_batched_tokens` 必须为正，`max_model_len` 必须为正或 `-1`。 +- 禁止：让 `dense_legacy` 因这些字段改变既有 pipeline parallel、cache 或 scheduler 配置；用 truthiness 把 `0` 误判为非法或显式 KV pin；在末端静默丢弃非默认值，或把文本 encoder 的长度字段未经模型语义确认当作 DiT KV 上限。 +- 验收：structured、legacy 和 direct 配置路径覆盖默认、非默认、非法值与显式 `0`；paged 路径断言字段进入 native cache/scheduler consumer，`-1` 触发 auto-fit；dense 路径断言原有并行、显存和 token budget 保持不变。 ^[PR #6094] + ### CONF-3a — 争议以展开后的最终配置为准 - 触发：CLI、deploy YAML、`base_config` overlay、platform 覆盖、per-stage override diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f34d181ac..820926702 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b283215c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 0a1846aa` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index d1e3fe3a5..8ad6455a4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222"] +sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index 5efb24695..4e1ea5de3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -163,3 +163,10 @@ confidence: high - 禁止：用只覆盖一个 session 的固定 floor 忽略声明 capacity；只为 self-KV 预算而漏算 cross-attention、scratch 或模型自有状态；使用 process-global KV pool registry；让各 TP rank 依据不同的本地 free-memory 计算不同 capacity 后继续 collective；把单次 TP replay 或能启动当作多 session/多 rank 预算正确性的证明。 - 验收：覆盖 LingBot 两个 resident window、DreamZero 声明 capacity 64 的预算封顶、LRU eviction、失败 forward 清理、commit 后 block 回收和显式 graph pool 输入；真实 TP 还要用统一的 min free-memory/all-reduce 或 rank-0 广播验证所有 rank 得到相同 capacity，并分别验证 paged/direct 数值与单副本路由。^[PR #5491] +## DIFF-4p — Scheduler-managed Diffusion KV 必须原子分配并绑定请求生命周期 + +- 触发：启用 `paged_scheduler`，或修改 Worker KV geometry discovery、显存 profile、Scheduler admission、CFG allocation、`NewRequestData`/DLO RPC envelope 和 request terminal cleanup。 +- 强制：模型加载后从每个 Worker 的 cache-enabled attention 生成 rank-local native `FullAttentionSpec`，以最大 per-rank profile batch 测量 KV headroom，再通过 `get_kv_cache_configs()`、`generate_scheduler_kv_cache_config()` 和 `resolve_kv_cache_block_sizes()` 构造 Worker/Scheduler 配置。Scheduler 侧只能以 native `KVCacheManager` 的薄 wrapper 为物理 owner；每个 CFG sequence 按完整首步 `seq_len` 以 `full_sequence_must_fit=True` 分配，先按空池所需 block 总数做 never-fit 检查，并在所有 sequence 成功前保持原子性。生成的 `DiffusionKVMetadata` 必须与请求绑定并随完整 `NewRequestData` envelope 传到 Worker；完成、取消、错误、pop 和 close 都必须释放整组 allocation，preemption 只保留既有 allocation。 +- 禁止：引入独立 block pool、KV spec、refcount 或 physical lifecycle；把部分 CFG 分配当作成功；把临时容量不足误报为永久失败，或让 never-fit 请求在 FIFO 头部无限阻塞；在 DLO DP wave 中拆开 request 与对应 metadata，或让 scheduler/busy-loop 异常逃逸而不唤醒请求流。 +- 验收：覆盖 dense 默认旁路、rank/spec/memory mismatch、profile 顺序、CFG partial rollback、临时压力返回 `None` 后 FIFO 重试、空池也放不下时只终止该请求、native allocation error、preemption metadata 保留，以及 finish/cancel/error/pop/close 全部释放；DLO 多 rank RPC 必须逐 envelope 保持 request 与 metadata 配对。 ^[PR #6094] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a98bab4bb..d7a7e9cc1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b283215c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0a1846aa` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b243583cd..d321d5126 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b283215c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0a1846aa` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 815d67777..4a1d34af6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b283215c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0a1846aa` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c21ee1fc4..f84763779 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b283215c` 验证存在 +- 源码校验：以上路径均已在 `main @ 0a1846aa` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2569b8337..9b87af8a1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b283215c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0a1846aa`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `199ef777` PR #6186 新增 `MiniMaxAI/MiniMax-Music3`。本轮仅沉淀多组件部署闭合、CFG 成对调度、companion 原子 admission 和 speech adapter 请求合同；不能据此推断音质、并发性能或端到端 bit-identical，模型专属 prompt/window/weights 细节未新增规则。 +本次相对 `b283215c` 本轮新增 paged_scheduler 的配置归一化、native KV 控制面原子分配，以及 HunyuanImage3 的 attention 注册与启动 profile 规则。它们只证明 Scheduler 控制面和模型 opt-in 边界；不得据此推断 Worker physical kv_caches、BlockTable、分页 attention 执行、prefix sharing、connector delayed-free 或 AR→DiT 页面传输已经实现。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 199ef777 \ - --to b283215c \ + --from b283215c \ + --to 0a1846aa \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md index 28e985ed4..b25ee72d8 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md @@ -1,10 +1,10 @@ --- title: "HunyuanImage3 开发规则" created: 2026-07-13 -updated: 2026-07-31 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, hunyuan-image3] -sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py] +sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py"] --- # HunyuanImage3 开发规则 @@ -129,3 +129,10 @@ sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/dif - **HY3-7c — 再查 VAE 前的 dtype 边界。** 条件图像素保持官方精度直到真实 VAE 边界；逐点比较进入 VAE 前的 tensor，不能用最终图片“看起来差不多”替代。 - **HY3-7d — 再查 router 和 top-k 精度。** 输出前缀已对齐后才核对 MoE router、top-k、softmax 和 reduction dtype；只提高某一层精度必须有 logit/token 差异证据，不能全模型盲目 fp32。 - **HY3-7e — 最后区分可修差异和架构差异。** 已关闭 processor、VAE 和 router 等可修边界后，再评估 TP、paged KV、fused kernel 和 reduction order；没有证据不得承诺 bit-exact，也不能用“架构噪声”掩盖仍可修的差异。 + +- **HY3-5g — ImageKVCacheManager 必须可被 paged KV discovery 注册且按最大图输入 profile** + - 触发：修改 HunyuanImage3 的 `ImageKVCacheManager`、image attention 注册、paged KV role/dtype，或为 `paged_scheduler` 调整启动 memory profile 与参考图数量。 + - 强制：`ImageKVCacheManager` 必须是 `nn.Module` 并以 `forward` 暴露内部 attention，使其出现在 `named_modules()` 而不把 dense cache state 写入 `state_dict()`；内部 attention 必须显式设置 `paged_kv_cache_role="primary"` 与 `paged_kv_cache_dtype=torch.bfloat16`，该 KV dtype 独立于权重加载 dtype。paged 启动 profile 必须经过模型 owner preprocessing，使用 1024x1024、启用 CFG 和 metadata 声明的最大参考图数量，当前 HunyuanImage3 为 3 张；普通 dense warmup 仍保持独立的 512x512、无 CFG 路径。 + - 禁止：保持 cache manager 为普通对象而期待 Worker spec discovery 找到嵌套 attention；把 paged KV 标记省略为默认 dense；只用一张参考图或关闭 CFG 生成 profile 后宣称容量覆盖最大请求；把 Scheduler-only `DiffusionKVRequest` 状态发送到 profile Worker execution。 + - 验收：模型测试断言 manager 是 `nn.Module`、attention identity 可从 `named_modules()` 读回且 `state_dict()` 为空；spec discovery 断言 layer、role、BF16 dtype 与 layout capability；profile 测试断言最大参考图数量、profile envelope 和 dense mode 的旁路行为。 ^[PR #6094] + From 89e930e4bdee8d7317c2b9adff473f7b1a46c7c4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:00:13 +0800 Subject: [PATCH 199/465] knowledge(vllm-omni): sync cd9ae66b (PR #6299) Upstream-Commit: cd9ae66b3a67b47e13092abcfe648b4ada86042e Upstream-PR: #6299 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9ceacb82e..a47dd211f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b283215c28d56d24b3f72a120b8f34222699aeda + previous_audited_sha: 0a1846aa458283a13c1a7b5a96ab585937f9726c audited_ref: main - audited_sha: 0a1846aa458283a13c1a7b5a96ab585937f9726c + audited_sha: cd9ae66b3a67b47e13092abcfe648b4ada86042e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 22313a492..4673a0c57 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0a1846aa458283a13c1a7b5a96ab585937f9726c`. The canonical machine baseline +cd9ae66b3a67b47e13092abcfe648b4ada86042e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bc21c3cac..98295650c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0a1846aa` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ cd9ae66b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 820926702..d1e704c17 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0a1846aa` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ cd9ae66b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d7a7e9cc1..1e7d77b00 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0a1846aa` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ cd9ae66b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index d321d5126..8c3e988aa 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0a1846aa` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ cd9ae66b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4a1d34af6..6973258fd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0a1846aa` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ cd9ae66b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f84763779..ad0a2ff1e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0a1846aa` 验证存在 +- 源码校验：以上路径均已在 `main @ cd9ae66b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9b87af8a1..9048f7227 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0a1846aa`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ cd9ae66b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b283215c` 本轮新增 paged_scheduler 的配置归一化、native KV 控制面原子分配，以及 HunyuanImage3 的 attention 注册与启动 profile 规则。它们只证明 Scheduler 控制面和模型 opt-in 边界；不得据此推断 Worker physical kv_caches、BlockTable、分页 attention 执行、prefix sharing、connector delayed-free 或 AR→DiT 页面传输已经实现。 +本次相对 `0a1846aa` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b283215c \ - --to 0a1846aa \ + --from 0a1846aa \ + --to cd9ae66b \ --repo \ --mode report-only ``` From d6e246b9726b2f51504359610a1513e837a46088 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:01:47 +0800 Subject: [PATCH 200/465] knowledge(vllm-omni): sync 02069a5b (PR #6287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 02069a5bbf95b4f88de1c1e1e9c0da7564e697ad Upstream-PR: #6287 Knowledge-Delta: Diffusion LoRA 内部导入改用 canonical upstream `LoRAConfig`，避免 ROCm 启动时兼容包初始化造成循环导入。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-upstream-compat.md | 12 ++++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a47dd211f..7a74d12c4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0a1846aa458283a13c1a7b5a96ab585937f9726c + previous_audited_sha: cd9ae66b3a67b47e13092abcfe648b4ada86042e audited_ref: main - audited_sha: cd9ae66b3a67b47e13092abcfe648b4ada86042e + audited_sha: 02069a5bbf95b4f88de1c1e1e9c0da7564e697ad # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4673a0c57..adb1d80a3 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -cd9ae66b3a67b47e13092abcfe648b4ada86042e`. The canonical machine baseline +02069a5bbf95b4f88de1c1e1e9c0da7564e697ad`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 98295650c..80015452a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ cd9ae66b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 02069a5b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d1e704c17..d88cd1ac7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ cd9ae66b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 02069a5b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md index 5c7e7de57..3cf2f5c8b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md @@ -1,10 +1,10 @@ --- title: "Diffusion upstream 兼容规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py] +sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py, "PR #6287"] confidence: high --- @@ -31,3 +31,11 @@ confidence: high 以进程能启动替代 Cache-DiT block discovery 验证。 - 验收：打开/关闭 Cache-DiT 都能匹配目标 block；测试冻结参数名与 return annotation，并让真实 lazy compile error 失败而非静默降级。^[PR #5976] + +## DIFF-4q — 内部 diffusion LoRA 导入必须避开兼容包初始化 + +- 触发：修改 diffusion LoRA 的 `LoRAConfig` 导入、ROCm 启动补丁或可能在 `vllm_omni` 初始化期间加载 diffusion registry 的依赖图。 +- 强制：内部 diffusion 模块必须从 canonical upstream 路径 `vllm.config.lora` 导入 `LoRAConfig`；`vllm_omni.config.lora` 仅保留为对外兼容 re-export，并避免在 diffusion 模块初始化期间触发 `vllm_omni.config.__init__`、pipeline registry 或尚未完成初始化的 diffusion 模块。 +- 禁止：假设导入兼容包的子模块不会执行父包初始化；在 ROCm platform patch 已进入 diffusion registry 时重新引入该兼容导入；把此修复泛化为改变 `LoRAConfig` 类型或配置语义。 +- 验收：ROCm 启动/pytest collection 在 diffusion 模块完成初始化前不再出现 circular-import；断言两个导入路径解析为同一 `LoRAConfig` 类，并回归 LoRA manager/utils 的正常导入及 CUDA/XPU 初始化路径。^[PR #6287] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1e7d77b00..b1dbd3ee4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ cd9ae66b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 02069a5b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8c3e988aa..a896957d8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ cd9ae66b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 02069a5b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6973258fd..ad1501b39 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ cd9ae66b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 02069a5b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ad0a2ff1e..86553ed1e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ cd9ae66b` 验证存在 +- 源码校验：以上路径均已在 `main @ 02069a5b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9048f7227..34ab1f007 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ cd9ae66b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 02069a5b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `0a1846aa` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `cd9ae66b` 本轮在 Diffusion upstream 兼容规则中新增 DIFF-4q，记录内部 LoRA 模块应绕过兼容 re-export、直接导入 `vllm.config.lora` 以避免 ROCm 启动期间触发循环导入。该规则不表示 CUDA/XPU 也存在同样启动路径，亦不改变对外兼容导入 API。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 0a1846aa \ - --to cd9ae66b \ + --from cd9ae66b \ + --to 02069a5b \ --repo \ --mode report-only ``` From 50ea92b7e4cfa930bad7c97f4d4d92381ba75d12 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:04:02 +0800 Subject: [PATCH 201/465] knowledge(vllm-omni): sync e1c5f59a (PR #6050) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: e1c5f59aab091b5aed864808d118903ceb77d6ac Upstream-PR: #6050 Knowledge-Delta: 新增 upstream structured config 复用的生命周期与 keyword-only 门禁，以及显式、动态、alias-aware 的 stage→EngineArgs 投影规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7a74d12c4..f632b4f78 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: cd9ae66b3a67b47e13092abcfe648b4ada86042e + previous_audited_sha: 02069a5bbf95b4f88de1c1e1e9c0da7564e697ad audited_ref: main - audited_sha: 02069a5bbf95b4f88de1c1e1e9c0da7564e697ad + audited_sha: e1c5f59aab091b5aed864808d118903ceb77d6ac # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index adb1d80a3..d4ca8b3b7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -02069a5bbf95b4f88de1c1e1e9c0da7564e697ad`. The canonical machine baseline +e1c5f59aab091b5aed864808d118903ceb77d6ac`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 80015452a..857bb1886 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 02069a5b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e1c5f59a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 523a290e5..b7dea63be 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050"] --- # vLLM-Omni 配置开发门禁 @@ -97,6 +97,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：把 `FLASHINFER` 视为原生 FlashInfer decode 的保证；依据八个 codebook 推断 attention 为 `q_len=8`；只修改一个 Higgs profile；或把 SM90/XQA capability 通过误认为 BF16 工作负载的性能结论。 - 验收：配置 schema、structured/legacy projection 测试均能读回 `{"use_trtllm_attention": false}`；解析三个 Higgs YAML 并断言 Stage 0 的 backend 与 nested pin 一致，最终日志为 `flashinfer-native`，并用同一栈 A/B 验证 TTFP/RTF 回到原生 FlashInfer 的范围。 ^[PR #6068] +### VOMNI-CFG-1l — upstream 配置复用必须保持 structured 边界与生命周期安全 + +- 触发：修改 `VllmOmniConfig` 的 structured stage config、复用 upstream `LoadConfig`/`CacheConfig`/`SchedulerConfig`/`ParallelConfig`，或新增 `CompilationConfig`、`ProfilerConfig` 等 upstream value object 字段。 +- 强制：只有语义、配置归属、生命周期和 EngineArgs projection 均一致时才复用 upstream contract；继承的公开 structured config 必须 keyword-only，`stage.compilation_config` 与 `stage.profiler_config` 使用具体 upstream 类型并在构造期物化 mapping；保留 Omni 自有字段、延迟的 model/platform/rank/port/backend 初始化和显式 projection 边界。 +- 禁止：用字段同名替代语义与 owner 审计；让 positional constructor 因 upstream 字段重排而静默改变含义；把继承字段的默认值当作显式 engine input；在 head process 执行终态初始化，或让 quantization 等未审计边界随 upstream contract 自动扩张。 +- 验收：覆盖 keyword-only 签名与 positional rejection、mapping 到具体 upstream object、预构造 object 的类型保留、scheduler/parallel 传输安全 derived fields 及 msgpack/asdict registration；用非默认字段证明最终 EngineArgs projection，且默认值不泄漏、未归属字段明确失败，legacy 与 structured 结果保持一致。 ^[PR #6050] + ### CONF-1a — 多 stage 共卡时 diffusion stage 必须显式设 gpu_memory_utilization - 触发：多 stage 模型 stage 共卡时，分布式测试在模型加载期 CUDA OOM。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d88cd1ac7..b3e4289f2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 02069a5b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ e1c5f59a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b1dbd3ee4..a01daf1b9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 02069a5b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e1c5f59a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a896957d8..89e7b009c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 02069a5b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e1c5f59a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ad1501b39..ed22c8ece 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 02069a5b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e1c5f59a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 86553ed1e..e13b93cca 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 02069a5b` 验证存在 +- 源码校验：以上路径均已在 `main @ e1c5f59a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 08580a37c..ce45557ee 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050"] confidence: high --- @@ -205,3 +205,10 @@ confidence: high locator 建无界共享 side table；把完整 base64/路径写日志。 - 验收：同长度覆盖、并发改写、named+inline、speaker cache 和 prefix hit 均不复用旧内容； 非法路径不 stat，日志只含截断值且命中真实 logger hierarchy。 ^[PR #5670] + +### SERV-10a — stage config 到 EngineArgs 的投影必须显式且由 owner 进程物化 + +- 触发：修改 `stage_init_utils.py` 的 typed stage→`EngineArgs` projection、upstream config 字段映射或 stage startup 的终态配置构造。 +- 强制：AR/generation 从 upstream config dataclass 与 `EngineArgs` 的交集生成 projection map，显式记录且只投影构造时提供的字段，并集中处理 `cache_dtype`→`kv_cache_dtype`、`policy`→`scheduling_policy`、`data_parallel_master_ip`→`data_parallel_address` 等 alias；`CompilationConfig`/`ProfilerConfig` 以具体 upstream object 复制传递，最终 `VllmConfig` 由 engine-owning process 物化，diffusion 保持既有固定 owner 与 projection surface。 +- 禁止：维护会随 upstream schema 漂移的静态 allowlist；把 inherited defaults、runtime-owned `worker_cls`/`distributed_executor_backend`、topology-owned `scheduler_cls` 或 private API-process 字段静默送入 EngineArgs；显式提供但没有合法 projection 的字段不得被丢弃，也不得在 head process 解析设备、rank、端口或 backend。 +- 验收：非默认 upstream 字段能在最终 EngineArgs 读回且 alias 正确，未显式提供的 inherited defaults 不出现；显式无 projection 字段抛出明确错误；typed projection 保留 compilation/profiler 的 concrete type，structured stage registration 可 msgpack 传输，并回归 legacy/structured effective parity。 ^[PR #6050] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 34ab1f007..a1bc52e81 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 02069a5b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e1c5f59a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `cd9ae66b` 本轮在 Diffusion upstream 兼容规则中新增 DIFF-4q，记录内部 LoRA 模块应绕过兼容 re-export、直接导入 `vllm.config.lora` 以避免 ROCm 启动期间触发循环导入。该规则不表示 CUDA/XPU 也存在同样启动路径，亦不改变对外兼容导入 API。 +本次相对 `02069a5b` 本轮沉淀两条边界：structured stage config 可在语义、归属和生命周期一致时复用 upstream vLLM contract；其 EngineArgs 投影必须只发送显式输入，并由 engine-owning process 完成终态物化。不要据此推断 typed VllmOmniConfig 路径已经完成生产 cutover；文档与测试变更不转化为规则。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from cd9ae66b \ - --to 02069a5b \ + --from 02069a5b \ + --to e1c5f59a \ --repo \ --mode report-only ``` From eb4d2daa3e1e041d8ae63ef39daa71ed9f4c60d2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:05:38 +0800 Subject: [PATCH 202/465] knowledge(vllm-omni): sync 4cd66787 (PR #6125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 4cd667875f983819414ca6a19cf612b58e6c138e Upstream-PR: #6125 Knowledge-Delta: 新增 CPU offload 下 text encoder 输入必须跟随 pipeline compute device 的 diffusion 执行规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f632b4f78..0a0373a64 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 02069a5bbf95b4f88de1c1e1e9c0da7564e697ad + previous_audited_sha: e1c5f59aab091b5aed864808d118903ceb77d6ac audited_ref: main - audited_sha: e1c5f59aab091b5aed864808d118903ceb77d6ac + audited_sha: 4cd667875f983819414ca6a19cf612b58e6c138e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d4ca8b3b7..582e15c30 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e1c5f59aab091b5aed864808d118903ceb77d6ac`. The canonical machine baseline +4cd667875f983819414ca6a19cf612b58e6c138e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 857bb1886..755468d9e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e1c5f59a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4cd66787` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b3e4289f2..1e2df087e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e1c5f59a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4cd66787` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 6217831a5..5f167d604 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125"] confidence: high --- @@ -124,6 +124,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl ## 质量阈值与资源辅助 +### DIFF-1t — CPU offload 下 text encoder 输入必须跟随 pipeline compute device + +- 触发：pipeline 启用 CPU offload，组件参数仍报告为 CPU，但 forward hook 会在执行前将 text encoder 权重移到 pipeline 的加速设备。 +- 强制：text encoder 组件初始化和 `encode_prompt` 输入都必须以 pipeline 的 `self.device` 为设备合同；输入 ids 不得通过 `next(text_encoder.parameters()).device` 推导设备，并保持 CUDA、XPU、CPU 等后端可用。 +- 禁止：因 offload 参数位于 CPU 就把 encoder 输入放到 CPU；硬编码 CUDA 设备；把 offload hook 已移动权重当作输入可以留在 CPU 的依据。 +- 验收：在启用 CPU offload 的 SDXL text-to-image 测试中，断言 text encoder 输入与 `self.device` 一致，并覆盖 XPU 及至少一个其他后端；确认 embedding lookup 不再出现 CPU/XPU 混用错误。^[PR #6125] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a01daf1b9..77809ca5f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e1c5f59a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4cd66787` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 89e7b009c..0be266632 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e1c5f59a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4cd66787` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ed22c8ece..f82ce65d8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e1c5f59a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4cd66787` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e13b93cca..79ed2591f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e1c5f59a` 验证存在 +- 源码校验：以上路径均已在 `main @ 4cd66787` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a1bc52e81..32835d194 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e1c5f59a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4cd66787`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `02069a5b` 本轮沉淀两条边界：structured stage config 可在语义、归属和生命周期一致时复用 upstream vLLM contract；其 EngineArgs 投影必须只发送显式输入，并由 engine-owning process 完成终态物化。不要据此推断 typed VllmOmniConfig 路径已经完成生产 cutover；文档与测试变更不转化为规则。 +本次相对 `e1c5f59a` 本轮将 SDXL 在 CPU offload 下的 text encoder 输入设备合同沉淀到 diffusion execution-parity 规则；不能据此推断所有模型的 offload 实现或已证明 XPU/CUDA 端到端数值与性能 parity。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 02069a5b \ - --to e1c5f59a \ + --from e1c5f59a \ + --to 4cd66787 \ --repo \ --mode report-only ``` From ad662baac318f47aee45bc5959018e06b6df092d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:07:32 +0800 Subject: [PATCH 203/465] knowledge(vllm-omni): sync c3de04e1 (PR #6322) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: c3de04e12777708cfb994958c614fc8708cf5225 Upstream-PR: #6322 Knowledge-Delta: 新增配置规则：pipeline registry 必须依赖轻量拓扑配置，避免 eager Pi0 runtime import 造成循环导入，同时保留 runtime re-export 兼容性。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 10 +++++++++- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 22 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0a0373a64..24978447f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e1c5f59aab091b5aed864808d118903ceb77d6ac + previous_audited_sha: 4cd667875f983819414ca6a19cf612b58e6c138e audited_ref: main - audited_sha: 4cd667875f983819414ca6a19cf612b58e6c138e + audited_sha: c3de04e12777708cfb994958c614fc8708cf5225 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 582e15c30..d0746df56 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4cd667875f983819414ca6a19cf612b58e6c138e`. The canonical machine baseline +c3de04e12777708cfb994958c614fc8708cf5225`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 755468d9e..ed833c3a7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4cd66787` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c3de04e1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index b7dea63be..b4f25e982 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py"] --- # vLLM-Omni 配置开发门禁 @@ -185,3 +185,11 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 强制：对会 eagerly 导入 `pipeline_registry`、`StageConfigFactory` 或其他重型 pipeline 依赖的公开配置符号使用模块级延迟解析；首次访问时通过 `importlib` 解析并缓存真实对象，同时保持 `__all__`、`__dir__` 与直接子模块导入的身份一致。导入链测试必须在干净解释器中先验证轻量配置导入不会预加载 registry/PI0，再验证公开导出仍解析到直接实现。 - 禁止：在 `vllm_omni.config` 包初始化阶段无条件导入会反向加载 `DiffusionOutput` 的 registry；用 import 顺序或已预加载模块掩盖循环；删除公开导出或让 lazy proxy 与直接导入得到不同对象。 - 验收：静态检查顶层导入不含重型 registry/factory，干净子进程按 `vllm_omni.config.lora` → `vllm_omni.diffusion.data` 顺序导入成功，并断言 `StageConfigFactory`、`register_pipeline` 与直接模块导出的对象相同；覆盖仍需用这些公开符号的调用方。^[PR #6293] + +### VOMNI-CFG-2b — pipeline registry 必须只依赖轻量拓扑配置 + +- 触发：修改 `pipeline_registry.py` 的模型拓扑注册，或配置/插件导入链可能在配置解析期间预加载模型 runtime。 +- 强制：将 registry 所需的静态 `PipelineConfig` 放入无 runtime 副作用的轻量模块；registry 只导入该模块，runtime pipeline 继续 re-export 同一对象以保持既有导入路径，并用干净子进程验证 registry 解析不加载 runtime。 +- 禁止：让 `pipeline_registry` eager import 模型 runtime、依赖已预加载模块或导入顺序掩盖循环；不得因拓扑模块轻量就改变模型 runtime 的实际加载语义。 +- 验收：隔离进程中导入并解析目标 registry key，断言对应 runtime module 不在 `sys.modules` 且返回的 `PipelineConfig` 身份与字段正确；再验证 runtime 的兼容导入路径仍 re-export 同一拓扑对象。^[PR #6322] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1e2df087e..0b7d4b3bf 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4cd66787` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c3de04e1` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 77809ca5f..deecaa4f0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4cd66787` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c3de04e1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0be266632..07604e355 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4cd66787` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c3de04e1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f82ce65d8..c0bce05d5 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4cd66787` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c3de04e1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 79ed2591f..9c029d952 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4cd66787` 验证存在 +- 源码校验：以上路径均已在 `main @ c3de04e1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 32835d194..f7a1bb0c4 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4cd66787`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c3de04e1`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `e1c5f59a` 本轮将 SDXL 在 CPU offload 下的 text encoder 输入设备合同沉淀到 diffusion execution-parity 规则；不能据此推断所有模型的 offload 实现或已证明 XPU/CUDA 端到端数值与性能 parity。 +本次相对 `4cd66787` 本轮将 PR #6322 的 Pi0 pipeline registry 导入边界沉淀到配置规则：registry 只依赖轻量拓扑模块，避免预加载 Pi0 runtime 形成循环导入。该隔离进程测试不能推断所有模型 runtime 导入链或平台运行时均已验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from e1c5f59a \ - --to 4cd66787 \ + --from 4cd66787 \ + --to c3de04e1 \ --repo \ --mode report-only ``` From abe75f08a8318b41b61bb862949264cd7cac7d44 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:08:51 +0800 Subject: [PATCH 204/465] knowledge(vllm-omni): sync 284e05c8 (PR #6279) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 284e05c88b7b46be9fae6d822bf22075840cbfbb Upstream-PR: #6279 Knowledge-Delta: 新增 DIFF-2t：将 DLO AllGather 的在线量化支持收窄为经验证的 per-tensor online FP8，并明确普通 loader、布局重建和启动 host memory 边界。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-checkpoint-loading.md | 8 ++++++++ .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 24978447f..509ea05c8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4cd667875f983819414ca6a19cf612b58e6c138e + previous_audited_sha: c3de04e12777708cfb994958c614fc8708cf5225 audited_ref: main - audited_sha: c3de04e12777708cfb994958c614fc8708cf5225 + audited_sha: 284e05c88b7b46be9fae6d822bf22075840cbfbb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d0746df56..ee6603710 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c3de04e12777708cfb994958c614fc8708cf5225`. The canonical machine baseline +284e05c88b7b46be9fae6d822bf22075840cbfbb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ed833c3a7..476de8bdf 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c3de04e1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 284e05c8` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0b7d4b3bf..d62b9c390 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c3de04e1` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 284e05c8` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index c38fcf67e..cc2aaf04d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -132,3 +132,11 @@ confidence: high - 触发：修改在线/runtime quantization 与 layerwise offload、DLO 权重加载、非连续 FP8 权重 flatten/repoint 或加载后处理流程。\n- 强制：在线量化层必须在消费完对应 checkpoint weight 后才可流式移回 CPU，并跳过已完成层的重复 `process_weights_after_loading`；继续加载其他模型权重前释放可复用的量化缓存。对非连续权重按物理 storage 顺序打包，记录 `shape`、`stride` 和实际 storage span，并在普通 layerwise、DLO prefetch 与 resident replay 中用 `torch.as_strided` 重建；runtime-created FP8 与 DLO 组合必须使用 no-AllGather 路径。\n- 禁止：对转置 Cutlass FP8 weight 使用逻辑 `.flatten()` 后再 `.view(shape)`，丢失物理布局；为调用幂等处理而把已量化 CPU 层重新搬回 accelerator；把 runtime-created FP8 宣称支持 sharded DLO AllGather。\n- 验收：CPU/mock 测试验证普通 layerwise、DLO prefetch 和 resident replay 的值与 stride 均保持，覆盖不同 storage span、streaming 顺序、已处理层跳过与缓存释放；online FP8 + DLO AllGather 必须 fail fast，no-AllGather 路径须完成加载并通过对应输出/资源 smoke。^[PR #5910] 相关执行流见 [Diffusion architecture](architecture.md)；component lifecycle 见 [component lifecycle 规则](rules-component-lifecycle.md)。 + +## DIFF-2t — DLO AllGather 只允许已验证的 per-tensor online FP8 布局 + +- 触发：修改 DLO+AllGather 与在线量化的兼容门禁、普通 loader fallback，或 FP8 权重/scale 的分片重建。 +- 强制：仅允许 `Fp8PerTensorOnlineLinearMethod` 进入 DLO+AllGather；由于 direct checkpoint mmap 不能生成运行时量化布局，必须先由普通 loader 完成 FP8 权重和 scale，再交给 DLO 按 dtype 分片，并按记录的 shape、stride 重建转置 Cutlass 权重。每个 rank 启动时可能暂存完整 FP8 模型，运行时才保留 DLO shard。 +- 禁止：把该 allowlist 扩展到未经验证的在线量化方法；绕过普通 loader 直接 mmap 在线量化权重；丢失非连续权重的物理 stride；将 transient 完整模型 host memory 峰值描述为运行时常驻开销。此前 DIFF-2s 对所有 runtime-created FP8 的 AllGather 禁止由本规则收窄覆盖。 +- 验收：loader 测试必须证明 allowlist 分支确实执行、普通 loader 在 DLO 前完成权重与 scale 处理，并对未验证 method fail fast；DLO 测试必须验证 FP8 weight/scale 的 dtype、值、shape 和转置 stride 重建。另覆盖 no-AllGather 路径及至少一次真实多 rank AllGather smoke；声明质量或性能 parity 仍需固定模型、硬件和 exact workload 的独立证据。^[PR #6279] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index deecaa4f0..0f4891334 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c3de04e1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 284e05c8` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 07604e355..3b93ce977 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c3de04e1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 284e05c8` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c0bce05d5..214bde966 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c3de04e1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 284e05c8` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9c029d952..db71f6700 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c3de04e1` 验证存在 +- 源码校验：以上路径均已在 `main @ 284e05c8` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f7a1bb0c4..06aa133fd 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c3de04e1`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 284e05c8`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `4cd66787` 本轮将 PR #6322 的 Pi0 pipeline registry 导入边界沉淀到配置规则：registry 只依赖轻量拓扑模块，避免预加载 Pi0 runtime 形成循环导入。该隔离进程测试不能推断所有模型 runtime 导入链或平台运行时均已验证。 +本次相对 `c3de04e1` 本轮记录了 PR #6279 对 diffusion DLO 加载合同的更新：仅 per-tensor online FP8 支持 DLO AllGather，并通过普通 loader 在分片前完成权重与 scale 的物化。不可据此推断其他在线量化方法、direct checkpoint mmap 或所有模型与硬件组合均已支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 4cd66787 \ - --to c3de04e1 \ + --from c3de04e1 \ + --to 284e05c8 \ --repo \ --mode report-only ``` From 45f0a35a5d19fd14fada531ee6667b6815cbac00 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:10:27 +0800 Subject: [PATCH 205/465] knowledge(vllm-omni): sync f208f907 (PR #6320) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: f208f907bc351dbb863e353ab5d996707bd4c339 Upstream-PR: #6320 Knowledge-Delta: 新增共享 diffusion RoPE 在 NPU complex64 频率采样中使用 `index_select` 的算子兼容性门禁。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 509ea05c8..2ea6a3a69 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c3de04e12777708cfb994958c614fc8708cf5225 + previous_audited_sha: 284e05c88b7b46be9fae6d822bf22075840cbfbb audited_ref: main - audited_sha: 284e05c88b7b46be9fae6d822bf22075840cbfbb + audited_sha: f208f907bc351dbb863e353ab5d996707bd4c339 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ee6603710..00e7c2859 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -284e05c88b7b46be9fae6d822bf22075840cbfbb`. The canonical machine baseline +f208f907bc351dbb863e353ab5d996707bd4c339`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 476de8bdf..0f87a2c7b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 284e05c8` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f208f907` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d62b9c390..57437387a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 284e05c8` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f208f907` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 5f167d604..a8d46c893 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320"] confidence: high --- @@ -131,6 +131,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：因 offload 参数位于 CPU 就把 encoder 输入放到 CPU；硬编码 CUDA 设备；把 offload hook 已移动权重当作输入可以留在 CPU 的依据。 - 验收：在启用 CPU offload 的 SDXL text-to-image 测试中，断言 text encoder 输入与 `self.device` 一致，并覆盖 XPU 及至少一个其他后端；确认 embedding lookup 不再出现 CPU/XPU 混用错误。^[PR #6125] +### DIFF-1u — NPU complex64 RoPE 索引必须使用 `index_select` + +- 触发：修改 `WanS2VRotaryPosEmbed` 或 `RotaryEmbeddingS2VGrid` 的频率表采样、complex64 索引，或 NPU RoPE 兼容路径。 +- 强制：对 `freqs_split`/`freqs` 的帧、空间高度和空间宽度频率表统一使用 `torch.index_select(..., 0, index)`；当 `f_o < 0` 时先对帧频率表执行 `conj()` 再选择，并保持既有 `view`、`expand`、`reshape` 与采样顺序。 +- 禁止：在 NPU 路径对 complex64 频率表使用 `tensor[index]` advanced indexing，触发不支持 `aclnnIndex`/`DT_COMPLEX64` 的算子；不得因替换索引而改变共轭顺序、采样位置或输出布局。 +- 验收：分别覆盖两个 RoPE 类的正向与共轭频率采样，断言 `index_select` 结果的 shape/value 与参考实现一致；在 NPU TP2 上运行 Wan2.2-S2V 端到端推理，确认不再因 `aclnnIndex`/`DT_COMPLEX64` 崩溃并产生视频输出。^[PR #6320] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0f4891334..97538a459 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 284e05c8` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f208f907` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3b93ce977..b250175fb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 284e05c8` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f208f907` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 214bde966..d6fb703fc 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 284e05c8` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f208f907` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index db71f6700..11501ff5d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 284e05c8` 验证存在 +- 源码校验：以上路径均已在 `main @ f208f907` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 06aa133fd..42450972b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 284e05c8`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f208f907`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `c3de04e1` 本轮记录了 PR #6279 对 diffusion DLO 加载合同的更新：仅 per-tensor online FP8 支持 DLO AllGather，并通过普通 loader 在分片前完成权重与 scale 的物化。不可据此推断其他在线量化方法、direct checkpoint mmap 或所有模型与硬件组合均已支持。 +本次相对 `284e05c8` 本轮将 PR #6320 的 NPU complex64 RoPE 索引约束写入共享 diffusion 规则 `DIFF-1u`。该规则只说明指定 RoPE 路径应使用 `index_select` 规避 NPU 算子限制，不能推断所有 NPU complex64 索引、其他模型或其他硬件后端均已支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from c3de04e1 \ - --to 284e05c8 \ + --from 284e05c8 \ + --to f208f907 \ --repo \ --mode report-only ``` From e37f8e99f92008af4ae9fee6008d0ceaf9e699fe Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:13:49 +0800 Subject: [PATCH 206/465] knowledge(vllm-omni): sync b4b907e7 (PR #5877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b4b907e7b60086c6428145f8dbab687314971dea Upstream-PR: #5877 Knowledge-Delta: 新增 DIFF-4r 与 EXEC-14a，固化 diffusion 配置 shim 的 upstream 字段覆盖和复合模型 typed AutoConfig 配置合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2ea6a3a69..cdb3d8141 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 284e05c88b7b46be9fae6d822bf22075840cbfbb + previous_audited_sha: f208f907bc351dbb863e353ab5d996707bd4c339 audited_ref: main - audited_sha: f208f907bc351dbb863e353ab5d996707bd4c339 + audited_sha: b4b907e7b60086c6428145f8dbab687314971dea # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 00e7c2859..5e5ccecf0 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f208f907bc351dbb863e353ab5d996707bd4c339`. The canonical machine baseline +b4b907e7b60086c6428145f8dbab687314971dea`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0f87a2c7b..4a85decff 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f208f907` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b4b907e7` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 57437387a..402c67e19 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f208f907` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b4b907e7` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index a8d46c893..881733ccd 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py"] confidence: high --- @@ -173,6 +173,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：对已消费 marker 的 multipart 使用有限超时；超时后重新 `dequeue` 并把后续 marker 或 inline 消息与迟到 payload 配对；仅把有限超时调大作为修复。 - 验收：单测断言接收调用使用 `indefinite=True`，并让 overflow payload 延迟超过任何有限超时后再发送 inline 消息，断言 worker 按 `overflow-1`、`inline-2` 顺序处理。当前 PR 的 0.1 秒延迟仍在旧 1 秒超时内，且 monkeypatch 的模块变量未被生产代码读取，不能单独证明旧实现会失败。^[PR #4845] +### DIFF-4r — diffusion vLLM 配置 shim 必须覆盖内嵌组件字段 + +- 触发：修改 `_DiffusionVllmModelConfig`，或在 diffusion pipeline 中初始化会读取 `current_vllm_config.model_config` 的 vLLM 内嵌模型组件。 +- 强制：配置 shim 必须覆盖真实 upstream consumer 读取的模型字段；对于 `LogitsProcessor` 等组件，`head_dtype` 必须显式存在，默认值 `None` 表示沿用模型 dtype，并在组件构造前置于当前 `VllmConfig` 上下文中。 +- 禁止：依赖 `SimpleNamespace` 或缺失属性后捕获 `AttributeError` 继续；只为单个模型补字段却不检查共享 diffusion 配置 shim 的其他内嵌 vLLM consumer；把一次初始化成功描述成跨平台兼容。 +- 验收：在 `_DiffusionVllmModelConfig` 包装的 `VllmConfig` 下构造最小 SenseNova LLM，断言 `model.logits_processor.head_dtype is None`，并对 upstream consumer 做字段 census；真实 diffusion 初始化与非 SenseNova 路径仍须回归。^[PR #5877] + ### DIFF-6a — request-level batching 必须由结构兼容键和请求隔离共同约束 - 触发：diffusion pipeline 宣布支持 request-level batching，或 scheduler/admission 增加模型条件、`extra_args` 结构与 request-level output split。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 97538a459..6fde8c003 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f208f907` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b4b907e7` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b250175fb..47b668f51 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f208f907` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b4b907e7` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index e1996ef21..4cd4e2c5a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py"] --- # Model Executor 规则 @@ -155,3 +155,10 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru 当 issue 已提供最终配置和失败堆栈、current source 可读时，先按 [架构职责锚点](architecture.md#当前源码职责锚点) 完成一轮窄调查，目标是在 owner 确定后两分钟内给出首次根因。超时仍未闭环时必须指出缺失证据，不能静默扩大到历史 commit、其他模型或环境猜测。 Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置日志加全局/per-stage 合并函数；一处启动前容量校验及其完整异常控制流；一处与日志一致的 worker 失败点。三段一致即可决定“配置触发 + fail-fast 缺陷”的主要修复位置，不再为首次结论读取 config factory、模型 pipeline、完整 deploy、spawn 实现或 tag diff；只有三段之间发生冲突时才补这些文件。 + +### EXEC-14a — 复合模型配置必须使用注册的 typed PretrainedConfig + +- 触发：新增复合模型的 transformer 配置、嵌套 `llm_config`/`vision_config`，或修改 `transformers_utils.configs` 的 AutoConfig 注册与模型加载路径。 +- 强制：为顶层和嵌套配置定义 typed `PretrainedConfig` 子类；从 checkpoint 加载时把嵌套 dict 提升为对应配置对象，并同步完成 `AutoConfig.register`、模块导出和 pipeline 的 `from_pretrained` 消费，默认值集中在 typed config 构造器中。 +- 禁止：在 pipeline 内手写 JSON/`SimpleNamespace` 和分散的 `.get` 默认值；只注册顶层 model type 却遗漏嵌套配置；让配置注册、导出和真实 consumer 使用不同的 schema。 +- 验收：local 与 HF 配置都能解析顶层及嵌套 model type，断言字段和默认值、AutoConfig 注册及模块导出一致，并用最小真实模型构造覆盖配置对象被 consumer 接收。^[PR #5877] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d6fb703fc..1d568f2cd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f208f907` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b4b907e7` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 11501ff5d..b6697861a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f208f907` 验证存在 +- 源码校验：以上路径均已在 `main @ b4b907e7` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 42450972b..ec00e0dc6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f208f907`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b4b907e7`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `284e05c8` 本轮将 PR #6320 的 NPU complex64 RoPE 索引约束写入共享 diffusion 规则 `DIFF-1u`。该规则只说明指定 RoPE 路径应使用 `index_select` 规避 NPU 算子限制，不能推断所有 NPU complex64 索引、其他模型或其他硬件后端均已支持。 +本次相对 `f208f907` 本轮新增 DIFF-4r 与 EXEC-14a，分别固化 diffusion vLLM 配置 shim 的 upstream 字段合同，以及复合模型 typed `PretrainedConfig` 的注册与嵌套配置提升。PR 的 tiny CPU 回归不能推断 SenseNova 全链路、跨硬件质量或所有模型的配置 parity；`t_eps` 的请求级迁移仍属 SenseNova 专有行为，未在共享页泛化。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 284e05c8 \ - --to f208f907 \ + --from f208f907 \ + --to b4b907e7 \ --repo \ --mode report-only ``` From 9efde8c208223690a0b85e3c81ac953d3c8a2ac3 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:15:07 +0800 Subject: [PATCH 207/465] knowledge(vllm-omni): sync 07ceb332 (PR #6329) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 07ceb3324a52061d6fb0cd4293b573783d287896 Upstream-PR: #6329 Knowledge-Delta: 新增 SERV-11a，规定 TTS、audio generation 与 diffusion chat 的用户输入日志仅在 request logging 开启时以受限 DEBUG 输出。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cdb3d8141..10c3b0dbb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f208f907bc351dbb863e353ab5d996707bd4c339 + previous_audited_sha: b4b907e7b60086c6428145f8dbab687314971dea audited_ref: main - audited_sha: b4b907e7b60086c6428145f8dbab687314971dea + audited_sha: 07ceb3324a52061d6fb0cd4293b573783d287896 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5e5ccecf0..80c536113 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b4b907e7b60086c6428145f8dbab687314971dea`. The canonical machine baseline +07ceb3324a52061d6fb0cd4293b573783d287896`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4a85decff..e91ba8cb6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b4b907e7` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 07ceb332` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 402c67e19..fee74102f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b4b907e7` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 07ceb332` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6fde8c003..6d8d5f900 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b4b907e7` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 07ceb332` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 47b668f51..b2fc911d1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b4b907e7` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 07ceb332` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1d568f2cd..9b335c494 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b4b907e7` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 07ceb332` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b6697861a..ab154c81c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b4b907e7` 验证存在 +- 源码校验：以上路径均已在 `main @ 07ceb332` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index ce45557ee..573950bdf 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329"] confidence: high --- @@ -212,3 +212,10 @@ confidence: high - 强制：AR/generation 从 upstream config dataclass 与 `EngineArgs` 的交集生成 projection map，显式记录且只投影构造时提供的字段，并集中处理 `cache_dtype`→`kv_cache_dtype`、`policy`→`scheduling_policy`、`data_parallel_master_ip`→`data_parallel_address` 等 alias；`CompilationConfig`/`ProfilerConfig` 以具体 upstream object 复制传递，最终 `VllmConfig` 由 engine-owning process 物化，diffusion 保持既有固定 owner 与 projection surface。 - 禁止：维护会随 upstream schema 漂移的静态 allowlist；把 inherited defaults、runtime-owned `worker_cls`/`distributed_executor_backend`、topology-owned `scheduler_cls` 或 private API-process 字段静默送入 EngineArgs；显式提供但没有合法 projection 的字段不得被丢弃，也不得在 head process 解析设备、rank、端口或 backend。 - 验收：非默认 upstream 字段能在最终 EngineArgs 读回且 alias 正确，未显式提供的 inherited defaults 不出现；显式无 projection 字段抛出明确错误；typed projection 保留 compilation/profiler 的 concrete type，structured stage registration 可 msgpack 传输，并回归 legacy/structured effective parity。 ^[PR #6050] + +### SERV-11a — 用户输入日志必须默认降级并受限输出 + +- 触发：修改 TTS、audio generation 或 diffusion chat serving 请求日志，以及 `request_logger` 或 `max_log_len` 的行为。 +- 强制：INFO 级别只记录 request ID、模型类型、voice clone、reference image 和参数等元数据；用户输入仅在 `self.request_logger` 存在时通过 `logger.debug` 输出，并按 `max_log_len` 预留日志前缀长度进行截断；未配置上限时使用 200 字符默认上限。 +- 禁止：在 INFO 日志中写入原始 `text` 或 `prompt`；无条件输出 DEBUG 用户内容；使用固定预览长度替代请求日志开关；在未配置 `max_log_len` 时输出无界用户输入。 +- 验收：覆盖 TTS、diffusion TTS、audio generation 和 diffusion chat，断言 INFO 不含用户内容；启用 `--enable-log-requests` 时 DEBUG 才输出内容，且配置与未配置 `max_log_len` 时均遵守截断上限。 ^[PR #6329] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ec00e0dc6..233b16f7c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b4b907e7`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 07ceb332`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `f208f907` 本轮新增 DIFF-4r 与 EXEC-14a，分别固化 diffusion vLLM 配置 shim 的 upstream 字段合同，以及复合模型 typed `PretrainedConfig` 的注册与嵌套配置提升。PR 的 tiny CPU 回归不能推断 SenseNova 全链路、跨硬件质量或所有模型的配置 parity；`t_eps` 的请求级迁移仍属 SenseNova 专有行为，未在共享页泛化。 +本次相对 `b4b907e7` 本轮将 TTS、audio generation 和 diffusion chat serving 路径的用户输入日志隐私门禁沉淀为 SERV-11a。该规则仅覆盖这些请求日志路径，不代表所有 serving 日志或其他接口已自动具备同等脱敏与截断行为。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from f208f907 \ - --to b4b907e7 \ + --from b4b907e7 \ + --to 07ceb332 \ --repo \ --mode report-only ``` From da8e712d7d11ad7d79f75bcb01ceb3c7f8f674ba Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:17:43 +0800 Subject: [PATCH 208/465] knowledge(vllm-omni): sync 9683b7f4 (PR #6283) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 9683b7f4b44997c20c4e865ec89b9775bb60a49b Upstream-PR: #6283 Knowledge-Delta: 新增 diffusion packed SwiGLU 的平台分派/精度门禁，并记录 MiniMax H3 DiT 的本地 activation 接入合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 10 +++++++++- 11 files changed, 30 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 10c3b0dbb..08d04787f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b4b907e7b60086c6428145f8dbab687314971dea + previous_audited_sha: 07ceb3324a52061d6fb0cd4293b573783d287896 audited_ref: main - audited_sha: 07ceb3324a52061d6fb0cd4293b573783d287896 + audited_sha: 9683b7f4b44997c20c4e865ec89b9775bb60a49b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 80c536113..dd83e9b92 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -07ceb3324a52061d6fb0cd4293b573783d287896`. The canonical machine baseline +9683b7f4b44997c20c4e865ec89b9775bb60a49b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e91ba8cb6..a9e010f48 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 07ceb332` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9683b7f4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index fee74102f..2510df115 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 07ceb332` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9683b7f4` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 881733ccd..1ad30c992 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py"] confidence: high --- @@ -138,6 +138,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：在 NPU 路径对 complex64 频率表使用 `tensor[index]` advanced indexing，触发不支持 `aclnnIndex`/`DT_COMPLEX64` 的算子；不得因替换索引而改变共轭顺序、采样位置或输出布局。 - 验收：分别覆盖两个 RoPE 类的正向与共轭频率采样，断言 `index_select` 结果的 shape/value 与参考实现一致；在 NPU TP2 上运行 Wan2.2-S2V 端到端推理，确认不再因 `aclnnIndex`/`DT_COMPLEX64` 崩溃并产生视频输出。^[PR #6320] +### DIFF-1v — diffusion packed SwiGLU 必须保持平台分派与精度边界 + +- 触发：共享 diffusion MLP 将 `F.silu(gate) * up` 替换为 packed SwiGLU fused activation，或新增平台专用 activation dispatch。 +- 强制：输入最后一维必须保持 `[gate, up]` packed 合同，输出形状为输入去半后的最后一维，并保持输入的 device/dtype。通过本地 `CustomOp` 和 `current_omni_platform` 分派；CUDA 设备才调用 `torch.ops._C.silu_and_mul`，NPU/XPU 保持 native fallback，其他平台必须有独立且已注册的实现。不会实际使用 fused op 的平台不得在构造时 eager resolve 该 op，尤其是 MUSA 应延迟解析。 +- 禁止：依赖 vLLM 通用 `SiluAndMul` 代替 diffusion 本地合同；把 `CustomOp` 存在误认为 NPU 已使用 `torch_npu.npu_swiglu`；让 MUSA 构造因缺少 `_C.silu_and_mul` 失败；把 fused kernel 的输出宣称为必然 byte-identical，忽略不同舍入边界。 +- 验收：CUDA 上以 native reference 对照 fused 输出的 shape、dtype、device 和数值，并 profile 确认每个目标 MLP 使用 fused kernel；NPU/XPU 断言 native fallback，MUSA 覆盖无 op 注册时的构造与 forward，若启用 NPU fused path 则单独验证 `torch_npu.npu_swiglu`。固定 checkpoint、输入、seed 和步数比较 H3 输出时使用明确数值容差，不能以本 PR 的 byte-identical 报告替代跨平台证据。^[PR #6283] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6d8d5f900..fdb319766 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 07ceb332` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9683b7f4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b2fc911d1..877f15a17 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 07ceb332` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9683b7f4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9b335c494..a21dd7c0f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 07ceb332` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9683b7f4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ab154c81c..c41cf852d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 07ceb332` 验证存在 +- 源码校验：以上路径均已在 `main @ 9683b7f4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 233b16f7c..d071558ed 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 07ceb332`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9683b7f4`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b4b907e7` 本轮将 TTS、audio generation 和 diffusion chat serving 路径的用户输入日志隐私门禁沉淀为 SERV-11a。该规则仅覆盖这些请求日志路径，不代表所有 serving 日志或其他接口已自动具备同等脱敏与截断行为。 +本次相对 `07ceb332` 本轮新增 MiniMax H3 DiT packed SwiGLU 的共享平台分派与模型接入规则，覆盖 `SiluAndMul`、gate/up 布局及 CUDA/native fallback。不得据此推断 NPU 已启用融合、跨平台 bit-exact parity 或通用性能收益。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b4b907e7 \ - --to 07ceb332 \ + --from 07ceb332 \ + --to 9683b7f4 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index ae343bde7..ccb938693 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283"] confidence: high --- @@ -193,3 +193,11 @@ confidence: high - 强制：NPU 路径使用一次 `F.linear(x, self.gate_up_proj.weight)` 生成 packed gate/up，再调用 `torch_npu.npu_swiglu(gate_up, dim=-1)`，最后复用既有 `down_proj`；gate/up 顺序和结果须等价于 `F.silu(gate) * up`，且补丁只作用于 NPU 目标 text MLP，CPU/CUDA 保持原路径。 - 禁止：在 NPU 上切分 packed 权重后发起两次 `F.linear` 或使用未融合的 SiLU 乘法；只直接调用 helper 而不验证生产注册链；把该优化扩展到 H3 DiT 或其他模型。 - 验收：NPU 测试以多个 packed shape 对照 split projection reference，并从真实 diffusion runner 初始化入口断言目标 `MiniMaxH3Qwen3VLTextMLP.forward` 已替换且重复初始化幂等；未通过生产入口的 helper 数值测试不能作为 hook 生效证据。^[PR #6167] + +## MMH3-4b — H3 DiT 的 packed SwiGLU 必须通过本地 activation 接入 + +- 触发：修改 MiniMax H3 `MiniMaxH3MLP`、`fc1` packed gate/up 输出或 DiT activation 实现。 +- 强制：`MiniMaxH3MLP` 必须把 `fc1` 的 packed 输出按 `[gate, up]` 顺序直接交给本地 `SiluAndMul`，再将结果交给既有 `fc2`；模型接入遵循 [DIFF-1v](../../components/diffusion/rules.md) 的平台与精度合同，且只改变 H3 DiT，不改变 Qwen3-VL text encoder 的独立 MLP 路径。 +- 禁止：在已选择 fused 路径后重新 `chunk` 并执行两次 native SiLU/乘法；用通用 vLLM activation 类替代本地 diffusion layer；把当前 NPU native fallback 描述成 H3 NPU fused support，或把一次 H3 性能/精度观察推广到其他 task、平台和拓扑。 +- 验收：H3 contract test 用 packed sentinel 验证 gate/up 顺序、半维输出及 `fc2` 输入，并分别覆盖 CUDA fused 与 NPU/XPU/native fallback；固定 checkpoint、seed、steps 和输入对照 native/fused 输出并设数值容差，同时以 kernel/profile 证据确认优化路径确实生效。^[PR #6283] + From 75473ecf7ae0d9852fabeec422d1937b4c3f9998 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:20:47 +0800 Subject: [PATCH 209/465] knowledge(vllm-omni): sync 14f2c16f (PR #6281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 14f2c16fb28e1e0c8615372675f2757466260ad8 Upstream-PR: #6281 Knowledge-Delta: 新增 H3 Triton modulation 的设备与连续布局门禁，以及 FP32 融合、CPU fallback 和操作顺序的可审计合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 9 ++++++++- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 08d04787f..9720f37bf 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 07ceb3324a52061d6fb0cd4293b573783d287896 + previous_audited_sha: 9683b7f4b44997c20c4e865ec89b9775bb60a49b audited_ref: main - audited_sha: 9683b7f4b44997c20c4e865ec89b9775bb60a49b + audited_sha: 14f2c16fb28e1e0c8615372675f2757466260ad8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index dd83e9b92..70b2d7735 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -9683b7f4b44997c20c4e865ec89b9775bb60a49b`. The canonical machine baseline +14f2c16fb28e1e0c8615372675f2757466260ad8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a9e010f48..d1d0d58b9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9683b7f4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 14f2c16f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2510df115..225ea6a59 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9683b7f4` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 14f2c16f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 1ad30c992..5af61f441 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py"] confidence: high --- @@ -145,6 +145,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：依赖 vLLM 通用 `SiluAndMul` 代替 diffusion 本地合同；把 `CustomOp` 存在误认为 NPU 已使用 `torch_npu.npu_swiglu`；让 MUSA 构造因缺少 `_C.silu_and_mul` 失败；把 fused kernel 的输出宣称为必然 byte-identical，忽略不同舍入边界。 - 验收：CUDA 上以 native reference 对照 fused 输出的 shape、dtype、device 和数值，并 profile 确认每个目标 MLP 使用 fused kernel；NPU/XPU 断言 native fallback，MUSA 覆盖无 op 注册时的构造与 forward，若启用 NPU fused path 则单独验证 `torch_npu.npu_swiglu`。固定 checkpoint、输入、seed 和步数比较 H3 输出时使用明确数值容差，不能以本 PR 的 byte-identical 报告替代跨平台证据。^[PR #6283] +### DIFF-1w — Triton 融合 kernel 必须有设备能力与连续布局门禁 + +- 触发：共享 diffusion 增加或修改 Triton fused op，尤其是 indexed modulation、in-place 输出、RMSNorm 融合，或被模型与 TeaCache extractor 复用的 kernel wrapper。 +- 强制：启动前同时检查 `HAS_TRITON`、`current_platform.is_cuda()`、输入/输出 tensor 的 CUDA device、支持的 dtype/shape 以及 kernel 所需的 row-major contiguous 布局；若列 stride 不是 1，必须回退或显式改用支持该 stride 的实现。非 CUDA、unsupported 或 strided 路径走 eager reference，并通过原生 `RMSNorm` 保留 NPU 的 `torch_npu.npu_rms_norm` 等平台分派。融合 reduction 与 elementwise 在 FP32 中完成，最终只在写回时转换到目标 dtype。 +- 禁止：只用 `x.is_cpu` 作为 Triton 门禁；让 NPU、XPU、MUSA 或其他非 CUDA tensor 进入 raw Triton launch；认为传入 row stride 就足以支持非连续列；对错误布局静默读写；或用 mock/wiring 结果宣称真实 kernel 数值 parity。 +- 验收：CPU/mock 覆盖 CUDA、NPU、XPU、MUSA、unsupported dtype/shape、空行和 contiguous/strided 输入，断言非 CUDA 路径不启动 Triton 且 NPU 保留 native RMSNorm；CUDA 目标硬件以 FP32 reference 核对值、dtype 和容差，并单独记录 kernel 数量/延迟。当前 PR 未新增该模块测试且 NPU CI 不覆盖 H3，在补齐前不得称为生产级跨平台支持。^[PR #6281] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index fdb319766..3ff8bf1fa 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9683b7f4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 14f2c16f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 877f15a17..6f7c8c4e5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9683b7f4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 14f2c16f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a21dd7c0f..dc52c0198 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9683b7f4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 14f2c16f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c41cf852d..efc1b5fcd 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9683b7f4` 验证存在 +- 源码校验：以上路径均已在 `main @ 14f2c16f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d071558ed..13268546c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9683b7f4`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 14f2c16f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `07ceb332` 本轮新增 MiniMax H3 DiT packed SwiGLU 的共享平台分派与模型接入规则，覆盖 `SiluAndMul`、gate/up 布局及 CUDA/native fallback。不得据此推断 NPU 已启用融合、跨平台 bit-exact parity 或通用性能收益。 +本次相对 `9683b7f4` 本轮新增共享 diffusion Triton 融合算子的设备与连续布局门禁，并补充 MiniMax H3 modulation 的操作顺序、精度与平台回退规则。不得由此推断 NPU/XPU/MUSA 已获支持、CUDA 结果 bit-exact，或性能数字可泛化。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 07ceb332 \ - --to 9683b7f4 \ + --from 9683b7f4 \ + --to 14f2c16f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index ccb938693..0f93dd894 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py"] confidence: high --- @@ -187,6 +187,13 @@ confidence: high - 禁止：在局部路径 gather `[S, 5376]` BF16 hidden state；绕过 `sp_input---local_sp_prepare` hook；只截取 text embedding 而不先运行完整 refiner；修改 Ulysses Q/K/V all-to-all、attention backend、packed-prefix 或 scheduler 语义；把约 21 倍 gather payload 降低写成端到端性能保证。 - 验收：模型合同测试须证明各 rank 的局部 multimodal rows 拼接等于完整 embedding、RoPE 行数与 span 对齐、所有 fallback 恢复旧路径，并断言 projection-before-gather 的 payload width 与最终 logits/target/condition rows 等价；性能和 video/audio quality 只能在固定 PR workload、硬件、steps、seed 与 gate 下复核，不能外推到其他 task 或拓扑。 ^[PR #6173] +## MMH3-1m — H3 modulation 融合必须保持操作顺序、精度与平台回退 + +- 触发：修改 MiniMax H3 的 AdaLN scale/shift、gated residual、`norm1`/`norm2` 融合、最终输出调制、Triton modulation wrapper 或 TeaCache extractor 的对应调用。 +- 强制：保持 H3 的操作顺序：`norm1` 后执行 indexed scale/shift，attention 后执行 gated residual 并紧接 `norm2` 与第二组 indexed affine，最终层继续在投影前调制；CUDA 融合路径使用 FP32 累加并在输出边界转换到目标 dtype。CPU fallback 必须保留旧的两步数值边界，即 RMSNorm 结果先转换为输入 dtype，再执行 affine；所有路径都必须按 `combined_indices` 逐行选择同一 shift、scale 和 gate。 +- 禁止：交换 residual、RMSNorm、affine 或 MLP 的顺序；在 CUDA 融合路径重新引入 BF16 中间舍入后宣称同一精度；让非 CUDA 平台绕过原生 RMSNorm backend；把 CPU fallback 的结果或一次 H3 性能观察描述成 CUDA bit-exact、跨平台 parity 或通用加速保证。 +- 验收：H3 contract test 以 eager 公式分别核对 `norm1` 调制、gated residual+`norm2` 调制和 final-layer 调制，覆盖 index reorder、空行、输出 dtype、CPU 两步 fallback 与 CUDA FP32 容差；NPU 测试必须确认使用 `RMSNorm.forward_npu`/`torch_npu.npu_rms_norm` 而非 Triton，并在固定 shape、steps、seed、硬件和 checkpoint 下单独验证 kernel 数量与性能。^[PR #6281] + ## MMH3-4a — H3 Qwen3-VL NPU 文本 MLP 必须使用 packed gate/up 与融合 SwiGLU - 触发：修改 MiniMax H3 Qwen3-VL text encoder 的 MLP、`gate_up_proj` packing、NPU fused operator 或对应 platform override。 From 54314af39fde89f5ed7b77ed6d99c727d292052a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:23:33 +0800 Subject: [PATCH 210/465] knowledge(vllm-omni): sync 14bd8f87 (PR #5531) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 14bd8f8772dfb2b6ccb2117d6b9078bbb2edbe66 Upstream-PR: #5531 Knowledge-Delta: 新增 WORLD/DP/HSDP 拓扑闭合、FSDP2 DeviceMesh 所有权及并行组初始化失败原子清理规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../configuration/rules-parallel-topology.md | 10 +++++++++- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-checkpoint-loading.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 30 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9720f37bf..cf336ff1c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9683b7f4b44997c20c4e865ec89b9775bb60a49b + previous_audited_sha: 14f2c16fb28e1e0c8615372675f2757466260ad8 audited_ref: main - audited_sha: 14f2c16fb28e1e0c8615372675f2757466260ad8 + audited_sha: 14bd8f8772dfb2b6ccb2117d6b9078bbb2edbe66 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 70b2d7735..1d96b9d9a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -14f2c16fb28e1e0c8615372675f2757466260ad8`. The canonical machine baseline +14bd8f8772dfb2b6ccb2117d6b9078bbb2edbe66`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d1d0d58b9..08bae14c1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 14f2c16f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 14bd8f87` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-parallel-topology.md b/knowledge/repos/vllm-omni/components/configuration/rules-parallel-topology.md index 0f02e93af..3f40a2214 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules-parallel-topology.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules-parallel-topology.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: [vllm_omni/config/composable_parallel/, vllm_omni/config/config_factory.py, vllm_omni/config/stage_config.py] +sources: [vllm_omni/config/composable_parallel/, vllm_omni/config/config_factory.py, vllm_omni/config/stage_config.py, "PR #5531"] confidence: high --- @@ -56,3 +56,11 @@ confidence: high load-balancer consumer；策略生效前若设备不足必须 fail fast。 部署 YAML 的展开顺序见 [deploy-yaml](deploy-yaml.md)；stage 执行合同见 [model-executor 规则](../model-executor/rules.md)。 + +## CONF-4e — WORLD、DP 与 HSDP 拓扑必须在实际设备数上闭合 + +- 触发：修改 `DiffusionParallelConfig`、`OmniStageDiffusionParallelConfig` 对 `WORLD_SIZE`、DP、HSDP、SP/CFG/TP/PP/EP 的解析，或修改 async stage 的设备展开。 +- 强制：普通并行必须满足 `WORLD_SIZE = TP × SP × PP × CFG × DP`；省略 DP 时保留 `None`，直到获得实际 `num_gpus`/WORLD 后推导，显式 DP 必须与推导值一致。HSDP 必须令普通 DP 为 1，并满足 `WORLD_SIZE = hsdp_replicate_size × hsdp_shard_size`；SP/CFG 只能复用同一 WORLD 拓扑，其乘积必须为 1 或 WORLD。解析 DP 后再生成 devices，并将 `num_gpus` 保留到最终 stage engine args。 +- 禁止：在配置解析期把省略的 DP 固定为 1 后据此生成不足的设备列表；接受与 WORLD 推导值不一致的显式 DP；允许 HSDP 与 TP、普通 DP、PP 或 EP 组合；只用 `num_gpus` 做校验却不传播到 stage 配置。 +- 验收：覆盖 direct、structured 和 async stage 路径；`num_gpus=8`、TP=2、SP=2 必须得到 DP=2、WORLD=8 和 devices `0`–`7`，省略 DP 的配置在运行时解析前仍保持未指定；覆盖 HSDP mesh 不匹配及不支持组合，并确认都在进程组创建前失败。^[PR #5531] + diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 225ea6a59..c4a228886 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 14f2c16f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 14bd8f87` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index cc2aaf04d..ea5d14983 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6279", "PR #6445", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6279", "PR #6445", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, "PR #5531"] confidence: high --- @@ -140,3 +140,10 @@ confidence: high - 禁止：把该 allowlist 扩展到未经验证的在线量化方法；绕过普通 loader 直接 mmap 在线量化权重；丢失非连续权重的物理 stride；将 transient 完整模型 host memory 峰值描述为运行时常驻开销。此前 DIFF-2s 对所有 runtime-created FP8 的 AllGather 禁止由本规则收窄覆盖。 - 验收：loader 测试必须证明 allowlist 分支确实执行、普通 loader 在 DLO 前完成权重与 scale 处理，并对未验证 method fail fast；DLO 测试必须验证 FP8 weight/scale 的 dtype、值、shape 和转置 stride 重建。另覆盖 no-AllGather 路径及至少一次真实多 rank AllGather smoke；声明质量或性能 parity 仍需固定模型、硬件和 exact workload 的独立证据。^[PR #6279] +## DIFF-2u — diffusion 并行状态必须保持单一拓扑所有权并原子清理 + +- 触发：修改 diffusion `parallel_state`、HSDP/FSDP2 `DeviceMesh`、SP/CFG/PP group、VAE patch parallel 或并行状态初始化与销毁。 +- 强制：`RankGenerator` 只表达真实正交的 TP/SP/PP/CFG/DP 轴；HSDP 不得作为额外轴，FSDP2 `DeviceMesh` 独占 replicate/shard 进程组；VAE tile parallel 直接从 WORLD 选 rank。初始化前验证 distributed 已启动且状态为空，组创建失败时必须清理本次已创建的 diffusion 与 vLLM 组；销毁时同时清空 vLLM `_PP` 等引用。 +- 禁止：重新创建冗余 `_DIT` 或 `_FS` 组；把 HSDP shard 当作 `RankGenerator` 的乘法维度；让 HSDP 继续依赖已移除的 fully-shard accessor；清理后保留已销毁的 pipeline group 或允许残留部分状态阻塞下一次初始化。 +- 验收：覆盖 HSDP standalone、HSDP+SP/CFG、VAE WORLD group、mesh 维度与实际 WORLD 不一致、初始化中途异常、重复初始化和销毁后重初始化；断言失败后无残留组，vLLM `_PP` 被置空，MiniMax H3 的 VAE/pipeline caller 使用 `get_world_group().device_group`，并以真实多 rank smoke 验证 DeviceMesh 与 collective 拓扑。^[PR #5531] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3ff8bf1fa..42c8c901e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 14f2c16f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 14bd8f87` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6f7c8c4e5..5eb2a9b03 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 14f2c16f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 14bd8f87` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index dc52c0198..f3ab30c41 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 14f2c16f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 14bd8f87` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index efc1b5fcd..c0c1dafd0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 14f2c16f` 验证存在 +- 源码校验：以上路径均已在 `main @ 14bd8f87` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 13268546c..d97c5c6cc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 14f2c16f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 14bd8f87`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `9683b7f4` 本轮新增共享 diffusion Triton 融合算子的设备与连续布局门禁，并补充 MiniMax H3 modulation 的操作顺序、精度与平台回退规则。不得由此推断 NPU/XPU/MUSA 已获支持、CUDA 结果 bit-exact，或性能数字可泛化。 +本次相对 `14f2c16f` 本轮新增配置并行拓扑与 diffusion 分布式并行状态门禁：明确 WORLD/DP/HSDP 的解析、设备传播、DeviceMesh 所有权及初始化失败清理。不能据此推断所有 HSDP 组合或真实多卡性能已获验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 9683b7f4 \ - --to 14f2c16f \ + --from 14f2c16f \ + --to 14bd8f87 \ --repo \ --mode report-only ``` From 21e6294d2f1180c45c53506add6b8d91b04a42d3 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:26:40 +0800 Subject: [PATCH 211/465] knowledge(vllm-omni): sync adf6d0c1 (PR #4061) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: adf6d0c1ea33f74cf5e8ae7860cee8c9547c6bac Upstream-PR: #4061 Knowledge-Delta: 新增 SANA-WM Stage-1 标准 Diffusers checkpoint 布局与 strict loading gate（DIFF-2v）。 --- adapters/vllm_omni/release_baseline.yaml | 8 ++++---- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-checkpoint-loading.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 23 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cf336ff1c..8ae431a11 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 14f2c16fb28e1e0c8615372675f2757466260ad8 + previous_audited_sha: 14bd8f8772dfb2b6ccb2117d6b9078bbb2edbe66 audited_ref: main - audited_sha: 14bd8f8772dfb2b6ccb2117d6b9078bbb2edbe66 + audited_sha: adf6d0c1ea33f74cf5e8ae7860cee8c9547c6bac # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 88 sha256: 96d973728f1d67a959121c4aa0d8ba86cff32043ec6f90458f48d1d766acb8af diffusion: - count: 65 - sha256: b0a814ade12af087b87d45e0a0583cf844456851b3dc052496bb996823b4bbb4 + count: 66 + sha256: cc2a3220f4808990f9a164c8944d3b8b7ace7217599ac975fb18e9719d53029a pipelines: count: 59 sha256: c2516b9fb6124aa0774a05390a6f05c08ed6998e287b03c8252d85338934933e diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1d96b9d9a..1ce7444e1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -14bd8f8772dfb2b6ccb2117d6b9078bbb2edbe66`. The canonical machine baseline +adf6d0c1ea33f74cf5e8ae7860cee8c9547c6bac`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 08bae14c1..20cb5ff64 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 14bd8f87` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ adf6d0c1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c4a228886..195aaea44 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 14bd8f87` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ adf6d0c1` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index ea5d14983..88b03a14e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6279", "PR #6445", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, "PR #5531"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6279", "PR #6445", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, "PR #5531", "PR #4061"] confidence: high --- @@ -147,3 +147,10 @@ confidence: high - 禁止：重新创建冗余 `_DIT` 或 `_FS` 组；把 HSDP shard 当作 `RankGenerator` 的乘法维度；让 HSDP 继续依赖已移除的 fully-shard accessor；清理后保留已销毁的 pipeline group 或允许残留部分状态阻塞下一次初始化。 - 验收：覆盖 HSDP standalone、HSDP+SP/CFG、VAE WORLD group、mesh 维度与实际 WORLD 不一致、初始化中途异常、重复初始化和销毁后重初始化；断言失败后无残留组，vLLM `_PP` 被置空，MiniMax H3 的 VAE/pipeline caller 使用 `get_world_group().device_group`，并以真实多 rank smoke 验证 DeviceMesh 与 collective 拓扑。^[PR #5531] +## DIFF-2v — SANA-WM Stage-1 checkpoint 必须保持标准 Diffusers 布局 + +- 触发：新增或修改 SANA-WM Stage-1 的 checkpoint resolution、Diffusers component source、HF download allow-pattern、transformer config/weight loading，或尝试直接接入 NVLabs 原始发布布局。 +- 强制：运行时只消费离线转换的标准 Diffusers 树：`model_index.json` 必须解析为 `SanaWmPipeline`，并同时提供 `transformer/config.json`、`transformer/diffusion_pytorch_model.safetensors`、`vae/config.json` 和 `vae/diffusion_pytorch_model.safetensors`；从 transformer config 构造 `SanaWmConfig`，Stage-1 download pattern 只包含这组必要文件，refiner 不进入该路径。 +- 禁止：用运行时 YAML parser、原始 NVLabs key remap、layout sniff 或宽泛下载模式弥补不兼容的 checkpoint；把命名为 `SanaWmTwoStagesPipeline` 的旧两阶段 repo 当作当前 Stage-1 registry 自动发现成功，或静默跳过缺文件、未知/重复/shape 不符权重。 +- 验收：local path 与 HF snapshot 都验证上述文件、`model_index.json` class 和 transformer config 读取；weight loader 对 unknown/duplicate/shape mismatch fail fast 并由 strict coverage 检查 expected keys；Stage-1 smoke 证明不下载/初始化 refiner，旧 layout 只能在显式且兼容的 class override 下运行。^[PR #4061] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 42c8c901e..f3d8789d4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 14bd8f87` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ adf6d0c1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5eb2a9b03..78a3de2c6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 14bd8f87` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ adf6d0c1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f3ab30c41..32e518b9c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 14bd8f87` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ adf6d0c1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c0c1dafd0..2a2a91161 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 14bd8f87` 验证存在 +- 源码校验：以上路径均已在 `main @ adf6d0c1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d97c5c6cc..470df86ee 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 14bd8f87`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ adf6d0c1`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `14f2c16f` 本轮新增配置并行拓扑与 diffusion 分布式并行状态门禁：明确 WORLD/DP/HSDP 的解析、设备传播、DeviceMesh 所有权及初始化失败清理。不能据此推断所有 HSDP 组合或真实多卡性能已获验证。 +本次相对 `14bd8f87` 本轮新增 SANA-WM Stage-1 checkpoint 的标准 Diffusers 布局与严格加载门禁。现有 DIFF-4i 已覆盖 registry/能力 metadata 路径；不能由本规则推断 SANA-WM 的 camera/action 请求格式、TP 数值等价、序列并行支持或 Stage-2 refiner 支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 14f2c16f \ - --to 14bd8f87 \ + --from 14bd8f87 \ + --to adf6d0c1 \ --repo \ --mode report-only ``` From 3827a793c59c754e49702bd1bec332e728e5c9cb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:30:18 +0800 Subject: [PATCH 212/465] knowledge(vllm-omni): sync a630b0c2 (PR #6119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: a630b0c2ec5e8ffd3be4ff490e4a85fe06f46536 Upstream-PR: #6119 Knowledge-Delta: 新增统一模型路径解析的缓存/下载失败合同、Ming 共享模块边界，以及 Ming-TTS speaker extractor 的预取与离线下载策略。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../model-executor/rules-loader-contract.md | 9 ++++++++- .../vllm-omni/components/model-executor/rules.md | 9 ++++++++- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../repos/vllm-omni/models/ming-omni-tts/rules.md | 11 +++++++++-- 12 files changed, 38 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8ae431a11..2f958d9ae 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 14bd8f8772dfb2b6ccb2117d6b9078bbb2edbe66 + previous_audited_sha: adf6d0c1ea33f74cf5e8ae7860cee8c9547c6bac audited_ref: main - audited_sha: adf6d0c1ea33f74cf5e8ae7860cee8c9547c6bac + audited_sha: a630b0c2ec5e8ffd3be4ff490e4a85fe06f46536 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1ce7444e1..6f16d617a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -adf6d0c1ea33f74cf5e8ae7860cee8c9547c6bac`. The canonical machine baseline +a630b0c2ec5e8ffd3be4ff490e4a85fe06f46536`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 20cb5ff64..f48364444 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ adf6d0c1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a630b0c2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 195aaea44..cc6bbb52e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ adf6d0c1` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ a630b0c2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f3d8789d4..8fae76520 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ adf6d0c1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a630b0c2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 78a3de2c6..2fd507fed 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ adf6d0c1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a630b0c2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md index af492a358..57447bd6a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910"] +sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119"] confidence: high --- @@ -57,3 +57,10 @@ confidence: high - 触发：修改共享 quantization factory、pipeline component 配置解析或 `quantization_config` 的全局/按组件语义。\n- 强制：普通 `QuantizationConfig` 必须原样解析到每个由 pipeline 构造且支持量化的组件；只有 `ComponentQuantizationConfig` 的显式 component map 可以缩小作用域。resolver、pipeline 和 quantizable layer 必须使用一致的 runtime prefix namespace，并仅让实际支持的 vLLM quantizable layer 消费配置。\n- 禁止：用 `hasattr(quant_config, "resolve")` 等隐式约定替代共享 resolver；因为组件属于同一模型就自动重写任意 `torch.nn` layer、embedding、norm、VAE 或其他不支持量化的模块；解析到单一组件后仍要求另一套 owner prefix。\n- 验收：CPU/mock 与真实构造测试分别覆盖 global config 同时命中 `transformer`/`text_encoder`、component map 只命中指定组件、未命中返回 `None`、最长 prefix 匹配及 unsupported layer 保持 checkpoint dtype；Flux2 与 MiniMax-H3 的 resolver 必须共同通过该合同。^[PR #5910] +## EXEC-2e — 模型路径解析必须统一本地、缓存与下载失败语义 + +- 触发：组件需要同时接受本地目录和 Hugging Face 模型引用，或新增配置、辅助权重、speaker extractor 等模型文件解析调用。 +- 强制：本地目录必须直接返回；`allow_download=True` 必须经 `download_weights_from_hf_specific` 并转发 `allow_patterns`、`cache_dir`、`require_all=True`；缓存模式必须使用 Hugging Face `snapshot_download(..., local_files_only=True)`。解析失败必须向上抛出，由调用方决定是否降级；需要兼容 ModelScope 时应传入 `allow_download=True` 或先使用已解析的本地目录。 +- 禁止：在调用方重新实现相反失败语义，绕过带锁和 ModelScope 支持的下载 helper，缓存未命中时静默返回原始 repo id，或为了单个辅助文件下载整个仓库。 +- 验收：覆盖本地目录短路、缓存命中、缓存未命中抛错、下载 helper 路由、`allow_patterns`/`cache_dir` 转发及 ModelScope 解析路径；调用方测试必须证明其选择的失败和下载策略。 ^[PR #6119] + diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 4cd4e2c5a..bacc80ca5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119"] --- # Model Executor 规则 @@ -162,3 +162,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：为顶层和嵌套配置定义 typed `PretrainedConfig` 子类；从 checkpoint 加载时把嵌套 dict 提升为对应配置对象，并同步完成 `AutoConfig.register`、模块导出和 pipeline 的 `from_pretrained` 消费，默认值集中在 typed config 构造器中。 - 禁止：在 pipeline 内手写 JSON/`SimpleNamespace` 和分散的 `.get` 默认值；只注册顶层 model type 却遗漏嵌套配置；让配置注册、导出和真实 consumer 使用不同的 schema。 - 验收：local 与 HF 配置都能解析顶层及嵌套 model type，断言字段和默认值、AutoConfig 注册及模块导出一致，并用最小真实模型构造覆盖配置对象被 consumer 接收。^[PR #5877] + +### EXEC-15a — Ming-family 可复用模块必须归一到 `common/ming` + +- 触发：第二个 Ming-family 模型复用或重命名 CFM、Aggregator、speaker embedding 等实现，或修改其模型侧构造参数。 +- 强制：可复用实现以 `vllm_omni/model_executor/models/common/ming/` 为唯一归属，`ming_tts` 与 `ming_flash_omni` 从该目录导入；保留 checkpoint-facing 的类和参数语义，模型侧必须显式传入真实 latent width，且 checkpoint 配置覆盖默认值。 +- 禁止：在任一 Ming 模型目录保留近似副本，让一个模型跨包导入另一个模型的可复用实现，或在重命名后遗留旧 import；不能依赖 shared `Aggregator` 的默认 `in_channels` 代替模型真实的 `latent_dim`。 +- 验收：共享模块测试断言 `Aggregator`、CFM、speaker extractor 等符号的 `__module__`，确认模型包没有本地重复类；用 Ming-flash 的真实 latent width 构造并断言输出 shape，同时运行 Ming-TTS 与 Ming-flash 的模块回归。 ^[PR #6119] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 32e518b9c..d42246144 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ adf6d0c1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a630b0c2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2a2a91161..5d4a662bc 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ adf6d0c1` 验证存在 +- 源码校验：以上路径均已在 `main @ a630b0c2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 470df86ee..89b843e1d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ adf6d0c1`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a630b0c2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `14bd8f87` 本轮新增 SANA-WM Stage-1 checkpoint 的标准 Diffusers 布局与严格加载门禁。现有 DIFF-4i 已覆盖 registry/能力 metadata 路径；不能由本规则推断 SANA-WM 的 camera/action 请求格式、TP 数值等价、序列并行支持或 Stage-2 refiner 支持。 +本次相对 `adf6d0c1` 本轮沉淀了统一模型路径解析合同、Ming-family 共享模块归属，以及 Ming-TTS 说话人抽取在引擎预取与离线预处理间的下载策略。不能据此推断所有调用方都应联网下载、所有 ModelScope 路径都绕过缓存，或模型专有常量也必须立即移动到 common/ming；文档、配方和测试变更不作为规则记录。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 14bd8f87 \ - --to adf6d0c1 \ + --from adf6d0c1 \ + --to a630b0c2 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/ming-omni-tts/rules.md b/knowledge/repos/vllm-omni/models/ming-omni-tts/rules.md index 438ec7fda..8ffe09182 100644 --- a/knowledge/repos/vllm-omni/models/ming-omni-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/ming-omni-tts/rules.md @@ -1,10 +1,10 @@ --- title: "Ming-Omni-TTS 规则" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #4341"] +sources: ["PR #4341", "PR #6119"] confidence: high --- @@ -53,3 +53,10 @@ confidence: high 共享 graph/eager 规则见 [Diffusion DIFF-1a](../../components/diffusion/rules.md)； 模型语义证据矩阵见 [model validation](../../review/guides/model-validation.md)。 + +## MING-3a — 说话人抽取必须区分引擎预取与离线下载 + +- 触发：修改 Ming-TTS 的 `SpeakerEmbeddingExtractor`、引擎模型预取、离线 speaker embedding 预处理，或调整 HF/ModelScope 模型路径策略。 +- 强制：引擎内的 `SpeakerEmbeddingExtractor` 默认使用 `allow_download=False`，依赖已预取且包含 `campplus.onnx` 的模型目录；引擎构造前的离线 `end2end` 提取必须显式传 `allow_download=True`，并限制解析到 `campplus.onnx`。 +- 禁止：在线请求路径在缓存未命中时隐式联网，离线引擎前处理使用 cache-only 却假定文件已经预取，或在 `VLLM_USE_MODELSCOPE` 下假定 HF 的 `local_files_only` 分支能够完成 ModelScope 首次解析。 +- 验收：覆盖本地已解析目录、缓存命中/未命中和离线显式下载三条路径；确认缺失 `campplus.onnx` 明确失败，ModelScope 预取后的目录走短路，并验证 serving 与离线提取都能产生 speaker embedding。 ^[PR #6119] From 92f5bde02d8d92eacba6437de194bf51be985b41 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:34:17 +0800 Subject: [PATCH 213/465] knowledge(vllm-omni): sync 26e00371 (PR #6241) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 26e003715c83cabebdc05ee3466b8359133f1074 Upstream-PR: #6241 Knowledge-Delta: 沉淀 MOSS-TTS Local 的固定形状异步输出、单 token prefill、compact top-k 与 Local Depth RoPE cache 验收边界，并补充 MOSS adapter max_new_tokens 覆盖的现有 serving 合同证据。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../serving/rules-engine-lifecycle.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../repos/vllm-omni/models/moss-tts/rules.md | 19 +++++++++++++++++-- 11 files changed, 31 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2f958d9ae..64a39a466 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: adf6d0c1ea33f74cf5e8ae7860cee8c9547c6bac + previous_audited_sha: a630b0c2ec5e8ffd3be4ff490e4a85fe06f46536 audited_ref: main - audited_sha: a630b0c2ec5e8ffd3be4ff490e4a85fe06f46536 + audited_sha: 26e003715c83cabebdc05ee3466b8359133f1074 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6f16d617a..d21554bc3 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -a630b0c2ec5e8ffd3be4ff490e4a85fe06f46536`. The canonical machine baseline +26e003715c83cabebdc05ee3466b8359133f1074`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f48364444..a421df67c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a630b0c2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 26e00371` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index cc6bbb52e..9b1a87687 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a630b0c2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 26e00371` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8fae76520..f06ae96bd 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a630b0c2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 26e00371` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2fd507fed..5204425c1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a630b0c2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 26e00371` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d42246144..1caef1595 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a630b0c2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 26e00371` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5d4a662bc..22edd83f3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a630b0c2` 验证存在 +- 源码校验：以上路径均已在 `main @ 26e00371` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 2ae56f132..d55ed5b1c 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 89b843e1d..83010efa0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a630b0c2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 26e00371`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `adf6d0c1` 本轮沉淀了统一模型路径解析合同、Ming-family 共享模块归属，以及 Ming-TTS 说话人抽取在引擎预取与离线预处理间的下载策略。不能据此推断所有调用方都应联网下载、所有 ModelScope 路径都绕过缓存，或模型专有常量也必须立即移动到 common/ming；文档、配方和测试变更不作为规则记录。 +本次相对 `a630b0c2` 本轮在 MOSS-TTS 模型规则中沉淀 Local talker 的固定形状异步输出、单 token prefill、compact top-k 与 Local Depth RoPE cache 合同，并为现有 TTS adapter sampling 规则补充 PR #6241 证据。不能据此推断所有模型都可启用异步输出、compact sampler 与全词表采样 seed/bit 等价，或将 PR 中吞吐与质量数据视为当前树的通用结果。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from adf6d0c1 \ - --to a630b0c2 \ + --from a630b0c2 \ + --to 26e00371 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/moss-tts/rules.md b/knowledge/repos/vllm-omni/models/moss-tts/rules.md index 42587248b..940616a92 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/rules.md @@ -1,10 +1,10 @@ --- title: "MOSS-TTS 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py] +sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, "PR #6241", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py"] confidence: high --- @@ -40,3 +40,18 @@ confidence: high 共享 loader 的 dtype/config 最小读取合同见 [Model Executor rules](../../components/model-executor/rules.md)。 + +## MOSSTTS-2a — Local talker 异步输出必须保持固定 shape 和 prefill 边界 + +- 触发：修改 MOSS-TTS Local talker 的 `make_omni_output`、`compute_logits`、`preprocess`、async chunk 或 CUDA Graph/async schedule。 +- 强制：只有在每个 request row 保持固定 shape 且状态更新可在 eager 阶段完成时启用 `use_async_omni_output`；`make_omni_output` 必须按原 batch 顺序保留每个 request 的 code row，包括全 `audio_pad_token_id` 的 stop row，并逐行生成 `_batch_should_continue`，再由 CPU 侧 `talker2codec_raw_async_chunk` 过滤全 pad row。`_omni_is_prefill` 为真时即使 span 只有一个 token 也必须走 embedding prefill，不生成 MTP/audio 输入，并保持 `ref_offset` 与 audio state 的既有更新语义。 +- 禁止：在 GPU 上使用 `codes[should_continue]` 或其他会发现动态输出 shape 的 boolean indexing；在模型侧提前丢弃全 pad row；仅凭 `span_len` 判断单 token prefill；或在 async output 模式下延迟仍会被下一步覆盖的模型状态更新。 +- 验收：mixed batch 同时覆盖有效 row 与 stop row，断言 row 数、shape、原 batch 对齐和 `_batch_should_continue` 值，并确认下游只过滤全 pad row；单 token prefill 断言 embedding 结果、无 `mtp_inputs` 且不推进 audio generation；async output/graph 路径验证不存在 host 动态 shape discovery，状态与输出均可正确交付。 ^[PR #6241] + +## MOSSTTS-2b — Local sampler 与 Depth RoPE cache 必须保持直接路径及数值边界 + +- 触发：修改 MOSS-TTS Local 的 `_sample_token`、Local Depth attention RoPE、`torch.compile`/CUDA Graph capture，或为这些优化增加运行时开关。 +- 强制：在 `_run_prefix` 编译或 talker capture 前，为每个 Local Depth attention 预先构造位置 `0..n_vq-1` 的非持久 RoPE cache（当前 Local v1.5 为 `0..11`），执行时只切片并让较短序列复用既有 allocation；compact top-k 必须在保留的 sorted candidates 上完成 nucleus filtering、softmax 和 multinomial，再将 compact index 映射回原始词表。 +- 禁止：在每个 depth step 内重复执行 `arange`/`einsum`/`cos`/`sin`，在 captured body 中构造或替换 cache，或用环境变量把这些直接实现伪装成可选路径；不得声称 compact sampler 与宽度 1024 的 multinomial 在 seed 或 bit 上等价，也不得忽略 top-k 边界 ties 的有限精度差异。 +- 验收：RoPE 测试与参考公式数值一致，较短 slice 保持 cache storage；capture 前确认完整 cache 已物化且执行体只使用切片。采样测试断言结果属于 retained top-k、极小 `top_p` 选择 argmax，并在无 ties 时核对分布语义；质量或回归记录必须明确 compact width 改变 RNG 映射后的非 seed/bit-equivalence 边界。 ^[PR #6241] + From 0ec2f2724f28e61710de183347e1b7a7224abbfb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:34:51 +0800 Subject: [PATCH 214/465] knowledge(vllm-omni): sync ac006707 (PR #6202) Upstream-Commit: ac0067078bd01200776b74e15ee7119153ab1af8 Upstream-PR: #6202 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 64a39a466..ca1502bbe 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a630b0c2ec5e8ffd3be4ff490e4a85fe06f46536 + previous_audited_sha: 26e003715c83cabebdc05ee3466b8359133f1074 audited_ref: main - audited_sha: 26e003715c83cabebdc05ee3466b8359133f1074 + audited_sha: ac0067078bd01200776b74e15ee7119153ab1af8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d21554bc3..cb5108b55 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -26e003715c83cabebdc05ee3466b8359133f1074`. The canonical machine baseline +ac0067078bd01200776b74e15ee7119153ab1af8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a421df67c..d5d6cf16b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 26e00371` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ac006707` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9b1a87687..18046d08f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 26e00371` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ac006707` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f06ae96bd..8295320d9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 26e00371` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ac006707` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5204425c1..c9c1a15af 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 26e00371` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ac006707` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1caef1595..e14b34d09 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 26e00371` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ac006707` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 22edd83f3..f8973e599 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 26e00371` 验证存在 +- 源码校验：以上路径均已在 `main @ ac006707` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 83010efa0..110d7c156 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 26e00371`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ac006707`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `a630b0c2` 本轮在 MOSS-TTS 模型规则中沉淀 Local talker 的固定形状异步输出、单 token prefill、compact top-k 与 Local Depth RoPE cache 合同，并为现有 TTS adapter sampling 规则补充 PR #6241 证据。不能据此推断所有模型都可启用异步输出、compact sampler 与全词表采样 seed/bit 等价，或将 PR 中吞吐与质量数据视为当前树的通用结果。 +本次相对 `26e00371` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from a630b0c2 \ - --to 26e00371 \ + --from 26e00371 \ + --to ac006707 \ --repo \ --mode report-only ``` From 6cb81e0a9afd1bf30350e49b159f2feb8afed0ae Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:36:44 +0800 Subject: [PATCH 215/465] knowledge(vllm-omni): sync 70995834 (PR #6274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 70995834040f488d63ed5c916fcee4e94268a902 Upstream-PR: #6274 Knowledge-Delta: 新增 MiniCPM-o 4.5 流式 Whisper cache API 兼容和 Token2Wav eager TorchDynamo 解包的审查规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../vllm-omni/models/minicpm-o-4-5/rules.md | 16 +++++++++++++++- 10 files changed, 28 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ca1502bbe..dfe9dd3d8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 26e003715c83cabebdc05ee3466b8359133f1074 + previous_audited_sha: ac0067078bd01200776b74e15ee7119153ab1af8 audited_ref: main - audited_sha: ac0067078bd01200776b74e15ee7119153ab1af8 + audited_sha: 70995834040f488d63ed5c916fcee4e94268a902 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index cb5108b55..ace13bb99 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ac0067078bd01200776b74e15ee7119153ab1af8`. The canonical machine baseline +70995834040f488d63ed5c916fcee4e94268a902`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d5d6cf16b..c1e4d52be 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ac006707` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 70995834` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 18046d08f..35ef9bac6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ac006707` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 70995834` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8295320d9..6c2ff8c6e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ac006707` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 70995834` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c9c1a15af..610d66836 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ac006707` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 70995834` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e14b34d09..09431b319 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ac006707` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 70995834` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f8973e599..20b33a7f5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ac006707` 验证存在 +- 源码校验：以上路径均已在 `main @ 70995834` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 110d7c156..aecf6b763 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ac006707`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 70995834`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `26e00371` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `ac006707` 本轮记录 MiniCPM-o 4.5 的流式音频缓存 API 兼容性，以及 Token2Wav 中 eager TorchDynamo 包装导致首个 duplex 响应停顿的模型专有约束。不能据此推断所有 Transformers 版本、所有 Token2Wav 后端或其他模型都适用，也不能将该 PR 的性能数字视为普遍基准。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 26e00371 \ - --to ac006707 \ + --from ac006707 \ + --to 70995834 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index ea42fc6cc..b73ef5b44 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274"] confidence: high --- @@ -100,6 +100,13 @@ confidence: high - 禁止：捕获整个 Code2Wav Stage 2 或依赖全局 `--enforce-eager` 代替 stage-scoped 配置；把 graph 开关放进 shared connector extra；让失败捕获继续复用可能已损坏的 allocator/RNG state；把 NPU graph 的 parity、缓存或性能结论外推到 HiFT、TRT、CUDA 或其他模型。 - 验收：解析 `minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml` 和 `minicpmo_4_5_3gpu.yaml` 的 NPU 最终 stage config，断言 Stage 0/1 的 `PIECEWISE`、Stage 2 的 outer eager 与 additional config；CPU 测试覆盖 timestep、exact dispatch、cache bucket、clone ownership、runtime precondition、MATH 选择和 fail-stop；A3 测试覆盖 uncached/cached eager parity。任何性能声明还必须记录 exact command、A3 硬件/驱动/torch-npu、warmup、测量轮数、并发、延迟/RTF 和峰值 NPU memory。^[PR #5604] +## MCPMO-1g — Token2Wav 的 eager TorchDynamo 包装必须在流式热路径中移除 + +- 触发：MiniCPM-o 4.5 的 Token2Wav/CosyVoice 流式路径使用 `torch.compile(backend="eager")` 包装 `forward_chunk`，并在新 chunk shape 上出现首响应停顿。 +- 强制：在 `BatchedToken2Wav` 初始化时，仅当方法存在 `_torchdynamo_orig_callable` 或 `__wrapped__` 时恢复原始 bound method；保留 estimator、encoder、HiFT 的其他路径及其运行时语义。 +- 禁止：让 eager backend 的 tracing/guard 构造进入每个未见 chunk shape 的实时响应；把该解包扩大为所有后端、模型或 CUDA Graph 路径的通用优化。 +- 验收：检查存在包装时初始化后 `forward_chunk` 已指向原始实现、无包装时保持不变；用 cold 与 warm 的多 chunk duplex 回归确认首个响应不再长时间停顿、后续响应持续产出音频，且 steady-state 行为未被改变。 ^[PR #6274] + ## MCPMO-2a — registry 使用 4.5 config/version predicate - 触发：pipeline auto-detection 看到通用 `MiniCPMO` architecture。 @@ -151,6 +158,13 @@ confidence: high 位置参数穿过 `_gradient_checkpointing_func`，补该 path 前不能假设训练/checkpoint 兼容。 ^[PR #5165] +## MCPMO-2d — Whisper 流式音频缓存必须兼容 current 与 legacy Transformers API + +- 触发：MiniCPM-o 4.5 Whisper/APM 音频编码器需要在 Transformers 版本变化后复用流式 chunk 的 attention cache。 +- 强制：根据 `self_attn.forward` 签名选择 `past_key_values` 或旧版 `past_key_value`；读取缓存长度时优先支持 `self_attention_cache.get_seq_length()`，兼容需要 layer 参数的实现，并回退到 legacy tuple 的 key shape；当当前 attention 返回值没有 cache 项时保留传入 cache。 +- 禁止：固定传入已被新 API 吞入 `**kwargs` 的旧参数名；调用已移除的 `get_usable_length()`；依赖 experimental 模块的全局 monkey patch 才能读取缓存；把当前 API 返回的两项结果误解为应清空已有 cache。 +- 验收：覆盖 legacy/current attention 参数名、`EncoderDecoderCache`/`DynamicCache`、legacy tuple 和两种 `get_seq_length` 签名；连续编码两个 chunk 时缓存长度正确增长，输出与无缓存编码有可观察差异，并通过 MiniCPM-o 4.5 streaming/duplex 回归验证首响应和中断行为。 ^[PR #6274] + ## MCPMO-3a — TTS stage 的 batch 能力与 runtime_info 消费一致 - 触发：talker/TTS stage 的 `max_num_seqs`、batching 或 runtime bridge。 From 7c2493c66f5954ad9f418a93b5c483270ef08bda Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:37:20 +0800 Subject: [PATCH 216/465] knowledge(vllm-omni): sync 8dc3d504 (PR #6347) Upstream-Commit: 8dc3d504d8bae73bd6a627b4b759a4dbe6cd35fb Upstream-PR: #6347 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index dfe9dd3d8..121b0288b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ac0067078bd01200776b74e15ee7119153ab1af8 + previous_audited_sha: 70995834040f488d63ed5c916fcee4e94268a902 audited_ref: main - audited_sha: 70995834040f488d63ed5c916fcee4e94268a902 + audited_sha: 8dc3d504d8bae73bd6a627b4b759a4dbe6cd35fb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ace13bb99..d26bdd0e7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -70995834040f488d63ed5c916fcee4e94268a902`. The canonical machine baseline +8dc3d504d8bae73bd6a627b4b759a4dbe6cd35fb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c1e4d52be..6d0aa40a4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 70995834` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8dc3d504` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 35ef9bac6..280173ea2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 70995834` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8dc3d504` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6c2ff8c6e..5422882f9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 70995834` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8dc3d504` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 610d66836..7de27938e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 70995834` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8dc3d504` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 09431b319..b4589b49b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 70995834` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8dc3d504` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 20b33a7f5..8b5e469b0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 70995834` 验证存在 +- 源码校验：以上路径均已在 `main @ 8dc3d504` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index aecf6b763..b9d865953 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 70995834`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8dc3d504`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `ac006707` 本轮记录 MiniCPM-o 4.5 的流式音频缓存 API 兼容性，以及 Token2Wav 中 eager TorchDynamo 包装导致首个 duplex 响应停顿的模型专有约束。不能据此推断所有 Transformers 版本、所有 Token2Wav 后端或其他模型都适用，也不能将该 PR 的性能数字视为普遍基准。 +本次相对 `70995834` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from ac006707 \ - --to 70995834 \ + --from 70995834 \ + --to 8dc3d504 \ --repo \ --mode report-only ``` From 9d807431160bb94dfabeff483ad45d34785475f5 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:37:54 +0800 Subject: [PATCH 217/465] knowledge(vllm-omni): sync c3f80502 (PR #6369) Upstream-Commit: c3f80502e00b564d7b74d43a5e21fbc6b9da2aed Upstream-PR: #6369 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 121b0288b..38823441d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 70995834040f488d63ed5c916fcee4e94268a902 + previous_audited_sha: 8dc3d504d8bae73bd6a627b4b759a4dbe6cd35fb audited_ref: main - audited_sha: 8dc3d504d8bae73bd6a627b4b759a4dbe6cd35fb + audited_sha: c3f80502e00b564d7b74d43a5e21fbc6b9da2aed # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d26bdd0e7..e2265ca51 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8dc3d504d8bae73bd6a627b4b759a4dbe6cd35fb`. The canonical machine baseline +c3f80502e00b564d7b74d43a5e21fbc6b9da2aed`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6d0aa40a4..0f6f5f3d5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8dc3d504` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c3f80502` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 280173ea2..296978051 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8dc3d504` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c3f80502` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5422882f9..235fb1cd6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8dc3d504` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c3f80502` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7de27938e..01635dfef 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8dc3d504` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c3f80502` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b4589b49b..02bc545ca 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8dc3d504` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c3f80502` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8b5e469b0..8acc58d94 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8dc3d504` 验证存在 +- 源码校验：以上路径均已在 `main @ c3f80502` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b9d865953..10503ab3c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8dc3d504`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c3f80502`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `70995834` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `8dc3d504` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 70995834 \ - --to 8dc3d504 \ + --from 8dc3d504 \ + --to c3f80502 \ --repo \ --mode report-only ``` From d5d37f955d33a94799647c8f3261a3b924b3b01d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:39:00 +0800 Subject: [PATCH 218/465] knowledge(vllm-omni): sync ea24deef (PR #6130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: ea24deefcce4d44339105e0fe6d243743b858074 Upstream-PR: #6130 Knowledge-Delta: 新增 FLUX.2-Klein 图像编辑 position-id 预拼接不可重复拼接的执行校验规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 38823441d..f674df8a3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8dc3d504d8bae73bd6a627b4b759a4dbe6cd35fb + previous_audited_sha: c3f80502e00b564d7b74d43a5e21fbc6b9da2aed audited_ref: main - audited_sha: c3f80502e00b564d7b74d43a5e21fbc6b9da2aed + audited_sha: ea24deefcce4d44339105e0fe6d243743b858074 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e2265ca51..d101d5e81 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c3f80502e00b564d7b74d43a5e21fbc6b9da2aed`. The canonical machine baseline +ea24deefcce4d44339105e0fe6d243743b858074`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0f6f5f3d5..dbe4a1397 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c3f80502` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ea24deef` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 296978051..53aa2cf56 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c3f80502` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ea24deef` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 5af61f441..f81b6a660 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130"] confidence: high --- @@ -152,6 +152,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：只用 `x.is_cpu` 作为 Triton 门禁；让 NPU、XPU、MUSA 或其他非 CUDA tensor 进入 raw Triton launch；认为传入 row stride 就足以支持非连续列；对错误布局静默读写；或用 mock/wiring 结果宣称真实 kernel 数值 parity。 - 验收：CPU/mock 覆盖 CUDA、NPU、XPU、MUSA、unsupported dtype/shape、空行和 contiguous/strided 输入，断言非 CUDA 路径不启动 Triton 且 NPU 保留 native RMSNorm；CUDA 目标硬件以 FP32 reference 核对值、dtype 和容差，并单独记录 kernel 数量/延迟。当前 PR 未新增该模块测试且 NPU CI 不覆盖 H3，在补齐前不得称为生产级跨平台支持。^[PR #6281] +### DIFF-1x — 预拼接 image position ids 不能在 denoise 分支重复拼接 + +- 触发：修改 FLUX.2-Klein 图像编辑输入准备、`latent_ids`/`image_latent_ids` 拼接或 denoising loop 的 position-id 传递。 +- 强制：若输入准备阶段已将 `image_latent_ids` 拼接进 `latent_ids`，后续 image-edit 分支必须直接复用已组装的 `latent_ids`；position-id 序列长度必须与 `torch.cat([latents, image_latents], dim=1)` 的 hidden-states 长度一致。 +- 禁止：在 denoising loop 再次执行 `torch.cat([latent_ids, image_latent_ids], dim=1)`，或仅凭张量可构造而忽略 image block 被重复计数导致的 shape mismatch。 +- 验收：对 FLUX.2-Klein 4B 与 9B 的图像编辑 warm-up/dummy run 和完整推理分别验证 position-id 长度等于 hidden-states 长度，并覆盖 image-edit、reference-image 分支及无图像输入路径。^[PR #6130] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 235fb1cd6..3e8294f73 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c3f80502` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ea24deef` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 01635dfef..61b585cf0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c3f80502` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ea24deef` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 02bc545ca..b547fc156 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c3f80502` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ea24deef` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8acc58d94..747dfd5d6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c3f80502` 验证存在 +- 源码校验：以上路径均已在 `main @ ea24deef` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 10503ab3c..f7f1fb69f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c3f80502`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ea24deef`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `8dc3d504` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `c3f80502` 本轮将 FLUX.2-Klein 图像编辑中 image position ids 的重复拼接约束沉淀到 diffusion 共享执行规则；不得据此推断其他 pipeline 的 position-id 布局或整体图像编辑兼容性。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 8dc3d504 \ - --to c3f80502 \ + --from c3f80502 \ + --to ea24deef \ --repo \ --mode report-only ``` From 4f12ad61196358acbd649b80946c3a56131bae39 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:41:30 +0800 Subject: [PATCH 219/465] knowledge(vllm-omni): sync efaf4276 (PR #5571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: efaf4276f93d3e0833198e97ba091ffb530b3dab Upstream-PR: #5571 Knowledge-Delta: 新增 XPU 异步 D2H 必须使用设备无关的 torch.Event 建立通用侧流等待依赖，避免图像数据竞争损坏。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-platform-backends.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f674df8a3..14b11ec1c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c3f80502e00b564d7b74d43a5e21fbc6b9da2aed + previous_audited_sha: ea24deefcce4d44339105e0fe6d243743b858074 audited_ref: main - audited_sha: ea24deefcce4d44339105e0fe6d243743b858074 + audited_sha: efaf4276f93d3e0833198e97ba091ffb530b3dab # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d101d5e81..806f2642a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ea24deefcce4d44339105e0fe6d243743b858074`. The canonical machine baseline +efaf4276f93d3e0833198e97ba091ffb530b3dab`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index dbe4a1397..0867deef0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ea24deef` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ efaf4276` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 53aa2cf56..9eec758a9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ea24deef` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ efaf4276` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3e8294f73..faf019052 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ea24deef` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ efaf4276` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 61b585cf0..fbc44b991 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ea24deef` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ efaf4276` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index 8ef976099..706179212 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293"] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py"] confidence: high --- @@ -19,6 +19,13 @@ confidence: high - 禁止：通过实例调用返回集合的 `valid_runtime_modes()` 冒充实例有效性校验，或因共享签名新增可选参数而让平台 override 产生接口漂移。 - 验收：对 base NPU runner 与 generation override 分别覆盖 `randomize_inputs` 的默认和显式路径，并用包含 `FULL_DECODE_ONLY` 等复合模式的正负用例断言 `is_valid_runtime_mode()` 的校验结果和签名 parity。 ^[PR #6096] +## EXEC-10b — XPU 异步 D2H 必须记录设备无关的 torch.Event + +- 触发：修改 `XPUOmniPlatform.record_device_event()`，或修改使用通用 `torch.Stream.wait_event()` 门控异步 D2H 的 XPU 输出路径。 +- 强制：在当前计算流上记录设备无关的 `torch.Event()`，再由通用 stream 等待该事件，使 accelerator hooks 建立计算流到侧流复制的依赖；保持与 CUDA、ROCm 平台一致。 +- 禁止：向通用 `torch.Stream.wait_event()` 传递 `torch.xpu.Event()`，或通过零散的 `xpu.synchronize()` 掩盖事件类型不兼容造成的复制竞态。 +- 验收：mock 测试断言 `torch.Event()` 被创建并记录且 `torch.xpu.Event()` 未被调用；XPU 硬件测试在计算流完成后让侧流执行 non-blocking D2H，等待侧流同步后断言 host tensor 全部保持预期值、无尾部图像损坏。 ^[PR #5571] + ## EXEC-12a — ROCm 分页注意力必须走 packed KV 的兼容 varlen 路径 - 触发：修改 `vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py` 的 ROCm 分页注意力、AITER/`flash_attn` varlen kernel、`block_table` 映射或 KV block layout。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b547fc156..c1acac7f2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ea24deef` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ efaf4276` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 747dfd5d6..a302de691 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ea24deef` 验证存在 +- 源码校验：以上路径均已在 `main @ efaf4276` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f7f1fb69f..69a72c9d0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ea24deef`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ efaf4276`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `c3f80502` 本轮将 FLUX.2-Klein 图像编辑中 image position ids 的重复拼接约束沉淀到 diffusion 共享执行规则；不得据此推断其他 pipeline 的 position-id 布局或整体图像编辑兼容性。 +本次相对 `ea24deef` 本轮将 PR #5571 的 XPU 异步 D2H 设备事件合同沉淀到 model-executor 平台后端规则。测试文件按权威路由属于 testing/ci，因 work-topic 超出范围而丢弃；不能据此推断其他平台或所有异步输出路径均已验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from c3f80502 \ - --to ea24deef \ + --from ea24deef \ + --to efaf4276 \ --repo \ --mode report-only ``` From 02a82acc7d2d4a7319bc7d544ccb572128247f6e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 07:42:30 +0800 Subject: [PATCH 220/465] knowledge(vllm-omni): sync 6f55a06f (PR #6270) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 6f55a06fab90509d94b34bfc682312ca6e55fea2 Upstream-PR: #6270 Knowledge-Delta: 本单元仅清理文档、示例、注释和错误消息；deploy_config 取代旧 stage 配置入口的既有合同已由 CONF-3b 覆盖，无新增 durable rule。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 14b11ec1c..58519b675 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ea24deefcce4d44339105e0fe6d243743b858074 + previous_audited_sha: efaf4276f93d3e0833198e97ba091ffb530b3dab audited_ref: main - audited_sha: efaf4276f93d3e0833198e97ba091ffb530b3dab + audited_sha: 6f55a06fab90509d94b34bfc682312ca6e55fea2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 806f2642a..b0bf69cac 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -efaf4276f93d3e0833198e97ba091ffb530b3dab`. The canonical machine baseline +6f55a06fab90509d94b34bfc682312ca6e55fea2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0867deef0..fe57e99c5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ efaf4276` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6f55a06f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9eec758a9..9c03d8731 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ efaf4276` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6f55a06f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index faf019052..7826b7b31 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ efaf4276` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6f55a06f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index fbc44b991..5142bb391 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ efaf4276` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6f55a06f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c1acac7f2..3631c63cf 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ efaf4276` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6f55a06f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a302de691..27e76539a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ efaf4276` 验证存在 +- 源码校验：以上路径均已在 `main @ 6f55a06f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 69a72c9d0..609370b89 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ efaf4276`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6f55a06f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `ea24deef` 本轮将 PR #5571 的 XPU 异步 D2H 设备事件合同沉淀到 model-executor 平台后端规则。测试文件按权威路由属于 testing/ci，因 work-topic 超出范围而丢弃；不能据此推断其他平台或所有异步输出路径均已验证。 +本次相对 `efaf4276` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from ea24deef \ - --to efaf4276 \ + --from efaf4276 \ + --to 6f55a06f \ --repo \ --mode report-only ``` From 03e9476f68ed7a3ca065d744b896f68f9392b4e6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:01:43 +0800 Subject: [PATCH 221/465] knowledge(vllm-omni): sync bd17618c (PR #5569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: bd17618c11cf9362043b9d43dc744f99dc52dfb6 Upstream-PR: #5569 Knowledge-Delta: 补充 XPU 平台兼容包装对 Omni AR 异步路径所需 stream、event、device 与 graph 接口的完整映射约束。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-platform-backends.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 58519b675..a62f62a31 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: efaf4276f93d3e0833198e97ba091ffb530b3dab + previous_audited_sha: 6f55a06fab90509d94b34bfc682312ca6e55fea2 audited_ref: main - audited_sha: 6f55a06fab90509d94b34bfc682312ca6e55fea2 + audited_sha: bd17618c11cf9362043b9d43dc744f99dc52dfb6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b0bf69cac..d488c226c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6f55a06fab90509d94b34bfc682312ca6e55fea2`. The canonical machine baseline +bd17618c11cf9362043b9d43dc744f99dc52dfb6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fe57e99c5..b3ac7d729 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6f55a06f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bd17618c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9c03d8731..7e8211209 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6f55a06f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bd17618c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7826b7b31..dcac8b1e1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6f55a06f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bd17618c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5142bb391..cdabcb24c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6f55a06f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bd17618c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index 706179212..7d35cb1ba 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py"] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py"] confidence: high --- @@ -26,6 +26,13 @@ confidence: high - 禁止：向通用 `torch.Stream.wait_event()` 传递 `torch.xpu.Event()`，或通过零散的 `xpu.synchronize()` 掩盖事件类型不兼容造成的复制竞态。 - 验收：mock 测试断言 `torch.Event()` 被创建并记录且 `torch.xpu.Event()` 未被调用；XPU 硬件测试在计算流完成后让侧流执行 non-blocking D2H，等待侧流同步后断言 host tensor 全部保持预期值、无尾部图像损坏。 ^[PR #5571] +## EXEC-10c — XPU CUDA 兼容包装必须覆盖异步运行时接口 + +- 触发：XPU 等非 CUDA 平台复用 Omni AR 异步输出或 warm-up 路径，而共享 runner 调用了 `torch.cuda` 的 stream、event、device 或 graph 接口。 +- 强制：在 `vllm_omni/platforms/xpu/utils.py::torch_cuda_wrapper()` 中集中把 CUDA-facing API 映射到 XPU；`current_stream`、`default_stream`、`stream`、`set_stream`、`set_device` 使用 `partial`，`Stream` 映射为 `torch.xpu.Stream`，`Event` 忽略 CUDA 专属 `blocking` 参数后构造 `torch.xpu.Event`；仅在 `supports_xpu_graph()` 为真时映射 graph、`CUDAGraph`、pool 和 capture 状态接口。 +- 禁止：为适配 XPU 修改共享 `gpu_ar_model_runner.py`，或只映射 stream 而遗漏 `set_stream`、`set_device`、event 与受支持的 graph 接口；不得让通用 `torch.Stream.wait_event()` 接收不兼容的专用事件类型，也不得在不支持 XPU graph 时无条件安装 graph shim。 +- 验收：XPU 平台测试进入 wrapper 后断言异步 runner 所需的 stream、event、device 和 graph 调用均路由到对应 XPU API，且不支持 graph 时保持禁用；Qwen3-TTS 等 AR-stage warm-up 与完整异步输出 smoke 在 XPU 上成功完成，同时 CUDA 路径和共享 runner 行为保持不变。^[PR #5569] + ## EXEC-12a — ROCm 分页注意力必须走 packed KV 的兼容 varlen 路径 - 触发：修改 `vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py` 的 ROCm 分页注意力、AITER/`flash_attn` varlen kernel、`block_table` 映射或 KV block layout。 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3631c63cf..7d7f40ef9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6f55a06f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bd17618c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 27e76539a..97fb2cfba 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6f55a06f` 验证存在 +- 源码校验：以上路径均已在 `main @ bd17618c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 609370b89..c55797f8d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6f55a06f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bd17618c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `efaf4276` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `6f55a06f` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from efaf4276 \ - --to 6f55a06f \ + --from 6f55a06f \ + --to bd17618c \ --repo \ --mode report-only ``` From aadac31e33586595909a47ccebad74fd29f19112 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:02:17 +0800 Subject: [PATCH 222/465] knowledge(vllm-omni): sync 47b18936 (PR #6208) Upstream-Commit: 47b18936766ea8ca97e030dce9c70f2e79315fa1 Upstream-PR: #6208 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a62f62a31..aa5c3d51e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6f55a06fab90509d94b34bfc682312ca6e55fea2 + previous_audited_sha: bd17618c11cf9362043b9d43dc744f99dc52dfb6 audited_ref: main - audited_sha: bd17618c11cf9362043b9d43dc744f99dc52dfb6 + audited_sha: 47b18936766ea8ca97e030dce9c70f2e79315fa1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d488c226c..311d56617 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bd17618c11cf9362043b9d43dc744f99dc52dfb6`. The canonical machine baseline +47b18936766ea8ca97e030dce9c70f2e79315fa1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b3ac7d729..4ee5861f7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bd17618c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 47b18936` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7e8211209..7ea8db760 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bd17618c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 47b18936` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index dcac8b1e1..2ec2c8683 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bd17618c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 47b18936` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cdabcb24c..c5ddda6b5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bd17618c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 47b18936` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7d7f40ef9..66cb6554c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bd17618c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 47b18936` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 97fb2cfba..7feca1bef 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bd17618c` 验证存在 +- 源码校验：以上路径均已在 `main @ 47b18936` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c55797f8d..864480080 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bd17618c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 47b18936`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `6f55a06f` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `bd17618c` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 6f55a06f \ - --to bd17618c \ + --from bd17618c \ + --to 47b18936 \ --repo \ --mode report-only ``` From e1fefd2efaa9fd462c0525245e19dedfa22cee23 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:06:58 +0800 Subject: [PATCH 223/465] knowledge(vllm-omni): sync 9c61d5b6 (PR #6346) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 9c61d5b67b2c57536b1b6246ef036d4e29bdb3d1 Upstream-PR: #6346 Knowledge-Delta: 新增 MiniCPM-o Talker codec 采样与 native duplex 分块终止、Code2Wav RelPos 分片、GPU/NPU duplex hook 接入和 stage min_tokens 上下文预算约束。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 9 +++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../serving/rules-engine-lifecycle.md | 9 +++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 +++---- .../vllm-omni/models/minicpm-o-4-5/rules.md | 23 ++++++++++++++++++- 12 files changed, 51 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index aa5c3d51e..e6c2004aa 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bd17618c11cf9362043b9d43dc744f99dc52dfb6 + previous_audited_sha: 47b18936766ea8ca97e030dce9c70f2e79315fa1 audited_ref: main - audited_sha: 47b18936766ea8ca97e030dce9c70f2e79315fa1 + audited_sha: 9c61d5b67b2c57536b1b6246ef036d4e29bdb3d1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 311d56617..322667dcd 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -47b18936766ea8ca97e030dce9c70f2e79315fa1`. The canonical machine baseline +9c61d5b67b2c57536b1b6246ef036d4e29bdb3d1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4ee5861f7..906b6d8a1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 47b18936` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9c61d5b6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7ea8db760..1d5dcfa85 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 47b18936` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 9c61d5b6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2ec2c8683..9b59c90eb 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 47b18936` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9c61d5b6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c5ddda6b5..7f7c95e80 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 47b18936` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9c61d5b6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index bacc80ca5..9d5de56be 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346"] --- # Model Executor 规则 @@ -115,6 +115,13 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru ## Async Omni output materialization +### EXEC-4d — GPU/NPU AR runner 必须在统一采样点接入 duplex hook + +- 触发：GPU/NPU AR runner 或 full-duplex sampling hook 修改 `prepare_duplex_sampling` 的接入与采样顺序。 +- 强制：GPU、NPU sibling runner 共同使用 `DuplexSamplingRunnerMixin`；初始化、模型加载解析和 state 更新走统一 helper，并在 `model_sample` 前对 prepared metadata 应用逐行 hook。 +- 禁止：只在 GPU `_sample` 接入而遗漏 NPU；在 `model_sample` 后调用 hook；复制平台专属实现或依赖 import side effect 建立 hook。 +- 验收：静态测试枚举全部 AR runner，断言 mixin、四个 hook site 与 sampler 前顺序；再覆盖 duplex 行映射和无 hook 模型的 no-op。 ^[PR #6346] + ### EXEC-5a — snapshot、live drain 与 fallback 必须构成同一个 output-cycle 合同 - 触发：修改 `GPUARModelRunner` async output、background builder、D2H copy、connector diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 66cb6554c..24cbc560e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 47b18936` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9c61d5b6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7feca1bef..5c5787f97 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 47b18936` 验证存在 +- 源码校验：以上路径均已在 `main @ 9c61d5b6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index d55ed5b1c..31256b506 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346"] confidence: high --- @@ -121,6 +121,13 @@ confidence: high - 禁止：在 `serving_speech.py` 增加 model-type 分支；静默忽略 `voice`、reference audio、temperature 或 `stream` 等不支持字段；把无 speaker 模型套用普通 TTS 的 voice 或 temperature 语义。 - 验收：通过真实 adapter detection 和 `/v1/audio/speech` 请求覆盖缺失/空 lyrics、缺失/空 caption、unsupported fields、长度边界、tokenizer 校验与非流式输出，并断言送入 engine 的实际 prompt、sampling 参数和 model type。 ^[PR #6186] +### SERV-5k — stage request 的 min_tokens 必须受剩余上下文约束 + +- 触发：orchestrator 用 `SamplingParams` 构造 stage request，且 prompt 已占用部分 `max_model_len`。 +- 强制：clone 参数后计算 `remaining=max_model_len-len(prompt_token_ids)`；`max_tokens is None` 时设为 `remaining`，并将 `min_tokens` 限制为 `max(0, remaining)`，不得改写调用方对象。 +- 禁止：只在 `max_tokens=None` 时处理 `min_tokens`；让 `check_stop` 因过大的最小 token 数永远早退；用静默 fallback 掩盖上下文已满。 +- 验收：覆盖剩余 46、50、0 token 的请求，断言最终 request 参数、原始参数不变，并回归普通 stage 的默认 max-token 行为。 ^[PR #6346] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 864480080..078c3e014 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 47b18936`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9c61d5b6`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `bd17618c` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `47b18936` 本轮新增 MiniCPM-o 4.5 Talker 的 vLLM codec 采样、native duplex 分块终止、Code2Wav RelPos 窗口规则，并记录 GPU/NPU AR runner 的共享 duplex sampling hook 与 stage 请求预算约束。未新增页面，因此无需注册索引；CI、benchmark 和测试文件仅作为验收证据处理。不得据此推断所有硬件平台均已实测通过，或将 nightly 性能数据视为回归门槛。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from bd17618c \ - --to 47b18936 \ + --from 47b18936 \ + --to 9c61d5b6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index b73ef5b44..6796d1405 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346"] confidence: high --- @@ -107,6 +107,13 @@ confidence: high - 禁止：让 eager backend 的 tracing/guard 构造进入每个未见 chunk shape 的实时响应；把该解包扩大为所有后端、模型或 CUDA Graph 路径的通用优化。 - 验收：检查存在包装时初始化后 `forward_chunk` 已指向原始实现、无包装时保持不变；用 cold 与 warm 的多 chunk duplex 回归确认首个响应不再长时间停顿、后续响应持续产出音频，且 steady-state 行为未被改变。 ^[PR #6274] +## MCPMO-1h — Code2Wav 编码窗口必须受 RelPos 与缓存预算约束 + +- 触发：修改 MiniCPM-o Code2Wav 的长 codec prefill、`forward_chunk`、RelPos PE、upsample 或 conformer cache。 +- 强制：按 PE `max_pos`、upsample stride、cache offset 和 lookahead 计算安全 token 窗口，限制单片上限并切分超长输入；非最终片保留 lookahead，调用 `forward_chunk` 前确保 PE 足够，按 batch 行合并音频并延续 cache。 +- 禁止：把数千 codec token 一次送入固定 RelPos PE；忽略 cache offset 或把非最终短片静默截断；用空音频掩盖窗口溢出。 +- 验收：单测覆盖预算计算与切片计划，fake Code2Wav 测试断言每片大小、`last_chunk` 和非空音频；不得从该修复推断通用性能提升。 ^[PR #6346] + ## MCPMO-2a — registry 使用 4.5 config/version predicate - 触发：pipeline auto-detection 看到通用 `MiniCPMO` architecture。 @@ -205,6 +212,13 @@ confidence: high 也没有 NPU concurrency 1/4 的 latency、吞吐或内存数据，因此不得宣称性能或质量不变/提升。 ^[PR #5792] +## MCPMO-3d — Talker codec 采样必须由 Stage 1 Sampler 与 codec 词表闭环 + +- 触发：修改 MiniCPM-o 4.5 Talker 的 codec 采样、词表、EOS、prompt penalty 或 Stage 1 sampling 参数。 +- 强制：`ConditionalChatTTSConfig` 暴露 `vocab_size=num_audio_tokens`、`eos_token_id=num_audio_tokens-1`；完整采样项由 Stage 1 `default_sampling_params` 提供，缺项初始化失败；`head_code`、`emb_code` 与 `codes.audio` 按采样 id 闭环，Sampler 前将 scheduler prompt 全置为 `vocab_size`。 +- 禁止：从 `tts_config` 回退或硬编码 codec knobs；把 Thinker token/embed 或 scheduler 占位符当作 codec history；丢失 sampled id 或让 EOS 不可达。 +- 验收：配置、pipeline EOS、prompt 不变、codec delta/empty EOS 单测通过，并用长文本真实 text+audio/ASR 检查完整输出。 ^[PR #6346] + ## MCPMO-4a — native duplex 保留可恢复 Stage0，VAD 只作显式策略 - 触发：MiniCPM-o native duplex 的连续音频、LISTEN/SPEAK handoff 或 server VAD interruption。 @@ -228,6 +242,13 @@ confidence: high - 验收：成功更新同时改变派生状态；资源不足时无部分 mutation；buffer/defer/reject 均不锁， append 成功才锁，mode flip 明确拒绝。 ^[PR #6318] +## MCPMO-4c — native duplex Talker 必须按 generate_chunk 预算终止 + +- 触发：修改 MiniCPM-o native duplex Talker 的 `generate_chunk`、codec EOS 或请求级音频状态。 +- 强制：每个 native duplex 请求按一次 chunk 的 26 个采样管理状态；turn 边界 `min_tokens=0`，中间 chunk `min_tokens=max_tokens=26`，已转发 25 帧后强制下一个采样为 EOS，仅把非 EOS code 交给 Code2Wav。 +- 禁止：只依赖 YAML `max_tokens` 维持 turn 边界；在中间 chunk 永久屏蔽 EOS；把终止采样再次作为普通 codec frame 输出。 +- 验收：覆盖 prefill 边界元数据、step 24/25 的 mask/force EOS、simplex 不误触发以及多请求 delta/finished 对齐。 ^[PR #6346] + ## MCPMO-5a — accuracy 与 duplex perf 证据必须绑定协议、硬件和实际 gate - 触发：修改 MiniCPM-o 4.5 Daily-Omni、Seed-TTS、realtime duplex 数据集/客户端、benchmark From dca4a81214513cba3aad1a70b8a0f3588980db31 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:09:46 +0800 Subject: [PATCH 224/465] knowledge(vllm-omni): sync 38cf9adc (PR #6345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 38cf9adc380f8bc7c73a6ca4463e625a1bad57d0 Upstream-PR: #6345 Knowledge-Delta: 新增 MMH3-3k：MiniMax H3 decoder tile 少于并行 rank 时统一退回 rank-local decode，防止空任务 collective hang。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../vllm-omni/models/minimax-h3/rules-deployment.md | 9 ++++++++- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e6c2004aa..019ae0ebe 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 47b18936766ea8ca97e030dce9c70f2e79315fa1 + previous_audited_sha: 9c61d5b67b2c57536b1b6246ef036d4e29bdb3d1 audited_ref: main - audited_sha: 9c61d5b67b2c57536b1b6246ef036d4e29bdb3d1 + audited_sha: 38cf9adc380f8bc7c73a6ca4463e625a1bad57d0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 322667dcd..7b0134a8c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -9c61d5b67b2c57536b1b6246ef036d4e29bdb3d1`. The canonical machine baseline +38cf9adc380f8bc7c73a6ca4463e625a1bad57d0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 906b6d8a1..bd7cca43e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9c61d5b6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 38cf9adc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1d5dcfa85..c39e2a992 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9c61d5b6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 38cf9adc` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9b59c90eb..82fbb441c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9c61d5b6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 38cf9adc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7f7c95e80..91c619809 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9c61d5b6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 38cf9adc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 24cbc560e..c3f686f2a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9c61d5b6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 38cf9adc` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5c5787f97..98bb9d2ac 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9c61d5b6` 验证存在 +- 源码校验：以上路径均已在 `main @ 38cf9adc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 078c3e014..dc7645809 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9c61d5b6`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 38cf9adc`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `47b18936` 本轮新增 MiniCPM-o 4.5 Talker 的 vLLM codec 采样、native duplex 分块终止、Code2Wav RelPos 窗口规则，并记录 GPU/NPU AR runner 的共享 duplex sampling hook 与 stage 请求预算约束。未新增页面，因此无需注册索引；CI、benchmark 和测试文件仅作为验收证据处理。不得据此推断所有硬件平台均已实测通过，或将 nightly 性能数据视为回归门槛。 +本次相对 `9c61d5b6` MiniMax H3 部署规则新增 decoder tile 少于并行 rank 时的 rank-local fallback。不可由此推断 encoder-side、其他 VAE 并行拓扑或性能收益已获支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 47b18936 \ - --to 9c61d5b6 \ + --from 9c61d5b6 \ + --to 38cf9adc \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 99cb466ca..8dad7f21c 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213"] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py"] confidence: high --- @@ -222,3 +222,10 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule - 禁止：让 backend 自行决定 checkpoint compatibility 或混合 DiT/非 DiT source 后只跳过部分 materialization；把 TP>1 ordinary-loader fallback、HSDP、online quantization 或其他 fail-closed 路径宣称具有 node-shared mmap 节省；把 resident layers 增加描述成减少 rank-local pinned host master。 - 验收：覆盖 dedicated/mixed source、strict non-DiT loading、AllGather shard 与 handle release、rank-local source retention、双 staging slot 上界、group size one 无 collective、TP>1 ordinary-loader fallback 及 unsupported-platform 不能静默丢弃 plan。PR 的 DP=2/TP=1 no-AllGather 与四 GPU topology smoke 只证明报告的 H3 配置和输出/资源观察；TP=2 行证明的是 ordinary-loader fallback，不是 shared-mmap memory guarantee。^[PR #6213] +## MMH3-3k — H3 VAE decoder 的 tile 不足 rank 时必须进入 rank-local decode + +- 触发：MiniMax-H3 修改 decoder tile 切分、`vae_patch_parallel_size`、parallel state，或低分辨率 preview/smoke 测试可能产生少于 tile group rank 数的 decoder tile。 +- 强制：`decode_latent` 必须按 latent shape 乘 `vae_ratio` 转为 pixel-space，并使用 checkpoint 的 decoder `split_tiles(..., True)` 计算 tile 数。当 `parallel_size > 1` 且 tile 数更少时，所有 rank 必须一致进入 `_rank_local_tiling`，将 checkpoint parallel state 设为 rank-local（`group_size`/`sp_size=1`、rank 为 0、process group 清空、parallel disabled）并将 `parallel_tiling=False`，使每个 rank 解码全部 tile；tile 数足够时保留并行路径，退出时完整恢复原状态。 +- 禁止：让无 tile 的 rank 抛错退出 collective、让其他 rank 留在 `all_gather`；不得使用 encoder grid 代替 decoder grid、让各 rank 依据不同条件分支，或把 rank-local fallback 描述成无性能代价或通用 VAE 并行支持。 +- 验收：CPU 回归必须实际调用 `decode_latent`，覆盖 decoder tile count 1、2、4、28、112 及 2/4、4/4、2/2、`parallel_size=1` 边界，断言 decode 观察到的 rank-local state，并验证正常与异常退出恢复完整 group state；4-rank 硬件复现中 2 tiles 必须正常退出，4 tiles 与 shipped `48x84` latent（28 tiles）必须保持并行路径。^[PR #6345] + From a49999f1e42ea629d139ee5ca2d2ca417cfd9308 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:11:32 +0800 Subject: [PATCH 225/465] knowledge(vllm-omni): sync b58feb14 (PR #6364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b58feb1420f347e5a5cdec98d2e6ea54b9be66d1 Upstream-PR: #6364 Knowledge-Delta: 补充 MUSA 默认兼容分派下 SwiGLU fused op 与 CPU native fallback 的审查合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 019ae0ebe..d0dfc975a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9c61d5b67b2c57536b1b6246ef036d4e29bdb3d1 + previous_audited_sha: 38cf9adc380f8bc7c73a6ca4463e625a1bad57d0 audited_ref: main - audited_sha: 38cf9adc380f8bc7c73a6ca4463e625a1bad57d0 + audited_sha: b58feb1420f347e5a5cdec98d2e6ea54b9be66d1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7b0134a8c..869264ebd 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -38cf9adc380f8bc7c73a6ca4463e625a1bad57d0`. The canonical machine baseline +b58feb1420f347e5a5cdec98d2e6ea54b9be66d1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bd7cca43e..d73027228 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 38cf9adc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b58feb14` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c39e2a992..660539a23 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 38cf9adc` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b58feb14` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index f81b6a660..9b5bcdab5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364"] confidence: high --- @@ -159,6 +159,13 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 禁止：在 denoising loop 再次执行 `torch.cat([latent_ids, image_latent_ids], dim=1)`，或仅凭张量可构造而忽略 image block 被重复计数导致的 shape mismatch。 - 验收：对 FLUX.2-Klein 4B 与 9B 的图像编辑 warm-up/dummy run 和完整推理分别验证 position-id 长度等于 hidden-states 长度，并覆盖 image-edit、reference-image 分支及无图像输入路径。^[PR #6130] +### DIFF-1y — MUSA 默认兼容路径的 SwiGLU 必须保留 fused op + +- 触发：修改 `vllm_omni/diffusion/layers/activation.py` 的 `SiluAndMul` device guard、`forward_cuda`/`forward_musa` 分派，或 MUSA 的 packed SwiGLU fused op。 +- 强制：`forward_cuda` 只在 `x.device.type == "cpu"` 时回退 `forward_native`；非 CPU 输入必须保留 packed `[gate, up]` 合同并进入 fused 路径。由于默认 `forward_musa` 转发到 `forward_cuda`，已提供兼容 op 的 MUSA 路径必须能够调用 `torch.ops._C.silu_and_mul`，同时保持输出 shape、dtype 和 device。 +- 禁止：使用 `x.device.type != "cuda"` 排除 MUSA；把通用 vLLM `SiluAndMul` 当作 diffusion 本地合同；或在 MUSA 构造阶段无条件 eager 解析可能不存在的 fused op。 +- 验收：CPU 测试断言 native fallback；MUSA mock/硬件测试断言 `forward_musa` 到达 fused op，并核对 packed 输入对应的输出 shape、dtype、device 和有界数值误差；另覆盖无 op 注册时的构造行为。该证据不能替代目标 MUSA 硬件上的完整模型 parity 或性能验证。^[PR #6364] + ### DIFF-3a — 质量阈值必须由完全相同的测试 case 产生 - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 82fbb441c..606119201 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 38cf9adc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b58feb14` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 91c619809..f3a833fcf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 38cf9adc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b58feb14` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c3f686f2a..350f0cd9f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 38cf9adc` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b58feb14` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 98bb9d2ac..1b34dbadd 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 38cf9adc` 验证存在 +- 源码校验：以上路径均已在 `main @ b58feb14` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index dc7645809..4d5099204 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 38cf9adc`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b58feb14`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `9c61d5b6` MiniMax H3 部署规则新增 decoder tile 少于并行 rank 时的 rank-local fallback。不可由此推断 encoder-side、其他 VAE 并行拓扑或性能收益已获支持。 +本次相对 `38cf9adc` 本轮将 MUSA SwiGLU 的 fused-op 分派与 CPU fallback 合同补充到 diffusion 共享规则中。该规则只说明分派边界与验证要求，不代表已证明所有 MUSA 硬件上的数值 parity、性能收益或其他平台支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 9c61d5b6 \ - --to 38cf9adc \ + --from 38cf9adc \ + --to b58feb14 \ --repo \ --mode report-only ``` From 2b4d7728204069654d07901f2a1092e082b795f1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:12:04 +0800 Subject: [PATCH 226/465] knowledge(vllm-omni): sync 480e10ba (PR #6356) Upstream-Commit: 480e10ba9c1cae673086da493e60c9d5edc92c22 Upstream-PR: #6356 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d0dfc975a..d82d2e5a5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 38cf9adc380f8bc7c73a6ca4463e625a1bad57d0 + previous_audited_sha: b58feb1420f347e5a5cdec98d2e6ea54b9be66d1 audited_ref: main - audited_sha: b58feb1420f347e5a5cdec98d2e6ea54b9be66d1 + audited_sha: 480e10ba9c1cae673086da493e60c9d5edc92c22 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 869264ebd..2ad18b0f8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b58feb1420f347e5a5cdec98d2e6ea54b9be66d1`. The canonical machine baseline +480e10ba9c1cae673086da493e60c9d5edc92c22`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d73027228..d11c8dd20 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b58feb14` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 480e10ba` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 660539a23..c869836bf 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b58feb14` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 480e10ba` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 606119201..29c311c65 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b58feb14` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 480e10ba` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f3a833fcf..2a4a5f824 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b58feb14` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 480e10ba` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 350f0cd9f..13c4ff5c7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b58feb14` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 480e10ba` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1b34dbadd..14f234ef8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b58feb14` 验证存在 +- 源码校验：以上路径均已在 `main @ 480e10ba` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4d5099204..a439faa3e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b58feb14`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 480e10ba`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `38cf9adc` 本轮将 MUSA SwiGLU 的 fused-op 分派与 CPU fallback 合同补充到 diffusion 共享规则中。该规则只说明分派边界与验证要求，不代表已证明所有 MUSA 硬件上的数值 parity、性能收益或其他平台支持。 +本次相对 `b58feb14` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 38cf9adc \ - --to b58feb14 \ + --from b58feb14 \ + --to 480e10ba \ --repo \ --mode report-only ``` From 20a8ba09c0d75fff2951730fd62d061ab2f94814 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:14:00 +0800 Subject: [PATCH 227/465] knowledge(vllm-omni): sync b57dc70e (PR #6360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b57dc70ec4b0dab61c69974a87d47fbcaa56564e Upstream-PR: #6360 Knowledge-Delta: 新增 resumable async-chunk 终态请求的 scheduler 所有权判定、定向 purge、计数恢复与 connector receiver 幂等回收规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/distributed/rules.md | 9 ++++++++- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d82d2e5a5..0e2e8aff5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b58feb1420f347e5a5cdec98d2e6ea54b9be66d1 + previous_audited_sha: 480e10ba9c1cae673086da493e60c9d5edc92c22 audited_ref: main - audited_sha: 480e10ba9c1cae673086da493e60c9d5edc92c22 + audited_sha: b57dc70ec4b0dab61c69974a87d47fbcaa56564e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2ad18b0f8..aefef0476 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -480e10ba9c1cae673086da493e60c9d5edc92c22`. The canonical machine baseline +b57dc70ec4b0dab61c69974a87d47fbcaa56564e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d11c8dd20..6e36a196c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 480e10ba` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b57dc70e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c869836bf..b78172b37 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 480e10ba` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b57dc70e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 29c311c65..1af1438f5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 480e10ba` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b57dc70e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 6e050e4f1..60c49810b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6033"] +sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6033", "PR #6360"] confidence: high --- @@ -56,6 +56,13 @@ confidence: high - 禁止：读取不存在的 `task["request_id"]`；用 `external_req_id` 代替 `self.requests` 的内部 key；发送失败后静默丢弃，或把已完成/已 abort 请求误当作仍可调度请求。 - 验收：覆盖发送抛异常、`connector.put` 返回 `False`、内部/外部 ID 不同、同一请求多次失败、并发 record/drain，以及请求在 sweep 前后离开 scheduler；日志必须包含正确 request ID 和失败原因，且 ledger 只被消费一次。^[PR #6033] +## DIST-1e — chunk connector 仅按隐藏队列所有权恢复状态并幂等回收 receiver + +- 触发：`OmniChunkTransferAdapter.finish_requests` 处理 resumable 请求的 `FINISHED_STOPPED` segment，且请求可能停在 `waiting_for_chunk_waiting_requests`、`waiting_for_chunk_running_requests` 或 `_held_non_active` 中。 +- 强制：只在 connector 隐藏队列实际持有请求时恢复 `requests_origin_status`；对每个目标统一调用幂等的 `cleanup_receiver`，清理 active stream、ready/exhausted/finished/cancelled/waiting 状态，并移除 chunk 等待队列中的已完成请求；请求已回到 scheduler 可见队列后不得再使用过期 origin status 覆盖当前状态。 +- 禁止：仅凭 `requests_origin_status` 存在就恢复状态；把 resumable `FINISHED_STOPPED` 当作普通 finished 请求而跳过 connector 回收；分散复制部分 receiver 清理逻辑导致重复释放或残留 active-stream 占用；用无关请求的队列状态扩大清理范围。 +- 验收：覆盖 connector 隐藏持有、scheduler `running`/`waiting`/`skipped_waiting` 持有、无 connector 所有权及 stale origin；断言隐藏队列、origin ledger、active streams 和 receiver 状态在重复 finish 后仍为空且 active-window 可被新请求重新占用。^[PR #6360] + ## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 - 触发：上游 MoE factory/oracle 或 `init_model_parallel_group` 增加 all-to-all manager 要求。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2a4a5f824..2c9308221 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 480e10ba` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b57dc70e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 13c4ff5c7..b2574daa2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 480e10ba` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b57dc70e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 812c8bca4..de5e838a9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6033"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6033", "PR #6360"] --- # Scheduler 规则 @@ -238,6 +238,13 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 禁止：只保护 full-payload 的 `WAITING_FOR_INPUT`；让负数、`nan` 或 `inf` 静默关闭 deadline；按 stream 总生命周期而非 chunk 间 stall 时间计时；对已离开 scheduler 的请求调用 `finish_requests`。 - 验收：覆盖有效值、显式零值、负数和非有限值解析，覆盖 dropped terminal chunk、producer 无响应、chunk 到达后重置、请求完成/abort 清理及同批健康请求继续运行；断言超时请求为 `FINISHED_ERROR`，并确认模块 reload 不污染后续测试。^[PR #6033] +## SCHED-5g — resumable async-chunk 终态清理必须以 live queue 所有权为准 + +- 触发：resumable async-chunk 请求在一个流式 segment 结束时进入 `FINISHED_STOPPED`，但仍由 `running`、`waiting`、`skipped_waiting` 或 connector 隐藏队列持有，随后 session close、取消或其他路径调用 `finish_requests`。 +- 强制：先物化可能被多层消费的 request-id iterator，并在 adapter 清理前快照 `skipped_waiting` 中承担流式等待计数的请求；只对仍有活队列所有权的目标 resumable 终态请求恢复状态，`skipped_waiting` 恢复为 `WAITING_FOR_STREAMING_REQ`，`running`/`waiting` 按实际队列对齐；running purge 必须限定本次 finish 集合，并确保 `_free_request`、coordinator 与 connector 清理恰好执行一次。 +- 禁止：把任意已完成请求重新打开；对脱离所有 live queue、可能等待 deferred block free 的终态请求调用释放；全局清空 running 中无关的 resumable segment；重复消费单遍 request-id iterator，或因非流式 skipped 请求错误减少 streaming counter。 +- 验收：AR 与 generation scheduler 均覆盖 hidden、`running`、`waiting`、`skipped_waiting`、脱离队列及无关 resumable 请求；断言状态、队列、计数、request map 和 active-stream capacity 正确，首次 finish 恰好释放、第二次无操作，并验证 generator request-id 输入。^[PR #6360] + ## SCHED-6a — async discard 的计数单位必须与 stale drain 一致 - 触发：上游 scheduler 改动异步占位、stale output 或 streaming segment replacement。 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 14f234ef8..8df6235a2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 480e10ba` 验证存在 +- 源码校验：以上路径均已在 `main @ b57dc70e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a439faa3e..8d06327a8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 480e10ba`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b57dc70e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b58feb14` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `480e10ba` 本轮新增 Scheduler 与 Distributed 两条规则：约束 resumable async-chunk 终态请求的队列所有权、计数恢复、定向清理及 connector 隐藏队列回收。不可据此推断 serving API、模型行为、final-input 生命周期或部署配置发生变化。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b58feb14 \ - --to 480e10ba \ + --from 480e10ba \ + --to b57dc70e \ --repo \ --mode report-only ``` From 1405ec5a194abd0d08bbcf300caf2a7472da4a54 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:15:37 +0800 Subject: [PATCH 228/465] knowledge(vllm-omni): sync 53df9fe6 (PR #6401) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 53df9fe62b3e05b388916351ecd758cb770e5cb4 Upstream-PR: #6401 Knowledge-Delta: 新增 DIFF-3c，记录 distributed VAE 必须使用专用 WORLD device group 以隔离 Wan spatial-shard 的 halo 通信与 collective。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-wan-spatial-shard.md | 12 ++++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0e2e8aff5..95c843d74 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 480e10ba9c1cae673086da493e60c9d5edc92c22 + previous_audited_sha: b57dc70ec4b0dab61c69974a87d47fbcaa56564e audited_ref: main - audited_sha: b57dc70ec4b0dab61c69974a87d47fbcaa56564e + audited_sha: 53df9fe62b3e05b388916351ecd758cb770e5cb4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index aefef0476..2df6ac5ee 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b57dc70ec4b0dab61c69974a87d47fbcaa56564e`. The canonical machine baseline +53df9fe62b3e05b388916351ecd758cb770e5cb4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6e36a196c..9ce041b3f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b57dc70e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 53df9fe6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b78172b37..c1842b48c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b57dc70e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 53df9fe6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-wan-spatial-shard.md b/knowledge/repos/vllm-omni/components/diffusion/rules-wan-spatial-shard.md index 6dd977777..2237ffac7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-wan-spatial-shard.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-wan-spatial-shard.md @@ -1,10 +1,10 @@ --- title: "Wan VAE spatial-shard 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #6062", vllm_omni/diffusion/distributed/autoencoders/wan_spatial_shard.py, tests/diffusion/distributed/test_wan_spatial_shard.py] +sources: ["PR #6062", vllm_omni/diffusion/distributed/autoencoders/wan_spatial_shard.py, tests/diffusion/distributed/test_wan_spatial_shard.py, "PR #6401"] confidence: high --- @@ -28,3 +28,11 @@ confidence: high 拼接所有固定 local shards 并 trim 后必须逐值重建输入。另覆盖 context/group rank mismatch 与 attention shrink/expand。CPU slicing tests 只证明边界数学；collective/halo correctness 仍需多 GPU height/width 对 single-process reference，生产 pipeline smoke 另验证真实 VAE patch/caller。^[PR #6062] + +## DIFF-3c — distributed VAE 通信必须使用专用 WORLD device group + +- 触发：修改 `DistributedVaeExecutor`、Wan VAE spatial-shard 的 collective/halo 通信或其 process group 选择。 +- 强制：分布式 VAE 通信必须使用 `get_world_group().device_group` 这一覆盖完整 worker WORLD 的专用 device process group，并通过该 group 获取 `world_size` 与 `rank`；不得使用默认 `group=None`。该组必须与 Wan spatial-shard 交错执行的 halo P2P/collective 通信隔离，避免共享默认 communicator 破坏 shard-boundary 数据。 +- 禁止：让 VAE tile collectives 或 P2P 复用默认 WORLD communicator；只凭单测中的 group 对象检查或 world_size=1 运行宣称 spatial-shard 已完成数值正确性验证。 +- 验收：mock/CPU 单测断言 `executor.group is get_world_group().device_group`，并覆盖通过该 group 查询 rank/world size；双 GPU 分别运行 height/width spatial-shard decode，与单进程 reference 对照并满足既有 `0.03` 最大差异容差。^[PR #6401] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1af1438f5..0f6ee5d74 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b57dc70e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 53df9fe6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2c9308221..3aa2d2213 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b57dc70e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 53df9fe6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b2574daa2..82ec533b4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b57dc70e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 53df9fe6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8df6235a2..d09cd72d2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b57dc70e` 验证存在 +- 源码校验：以上路径均已在 `main @ 53df9fe6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8d06327a8..1c41b05ab 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b57dc70e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 53df9fe6`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `480e10ba` 本轮新增 Scheduler 与 Distributed 两条规则：约束 resumable async-chunk 终态请求的队列所有权、计数恢复、定向清理及 connector 隐藏队列回收。不可据此推断 serving API、模型行为、final-input 生命周期或部署配置发生变化。 +本次相对 `b57dc70e` 本轮将 PR #6401 的 distributed VAE 通信隔离要求沉淀到 Wan spatial-shard 规则，并补充来源；这只说明专用 WORLD communicator 与该回归路径的合同，不代表所有 VAE、硬件或并行组合都已获得数值或性能支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 480e10ba \ - --to b57dc70e \ + --from b57dc70e \ + --to 53df9fe6 \ --repo \ --mode report-only ``` From c3361e39e433c699d3c5ec5d35c31812ceb9c1fc Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:17:13 +0800 Subject: [PATCH 229/465] knowledge(vllm-omni): sync 48e4f6a3 (PR #6397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 48e4f6a38504bc236a28d06e7c2837594c64e46f Upstream-PR: #6397 Knowledge-Delta: 新增 MCPMO-1i：MiniCPM-o 4.5 Code2Wav 初始化仅在 Flow 存在 encoder 时执行 TorchDynamo 解包。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md | 9 ++++++++- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 95c843d74..e5f31f0dc 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b57dc70ec4b0dab61c69974a87d47fbcaa56564e + previous_audited_sha: 53df9fe62b3e05b388916351ecd758cb770e5cb4 audited_ref: main - audited_sha: 53df9fe62b3e05b388916351ecd758cb770e5cb4 + audited_sha: 48e4f6a38504bc236a28d06e7c2837594c64e46f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2df6ac5ee..5e1b7d2b4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -53df9fe62b3e05b388916351ecd758cb770e5cb4`. The canonical machine baseline +48e4f6a38504bc236a28d06e7c2837594c64e46f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9ce041b3f..a96c30af5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 53df9fe6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 48e4f6a3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c1842b48c..105b9a45f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 53df9fe6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 48e4f6a3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0f6ee5d74..911fabf7a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 53df9fe6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 48e4f6a3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3aa2d2213..8db288279 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 53df9fe6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 48e4f6a3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 82ec533b4..65ff76f34 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 53df9fe6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 48e4f6a3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d09cd72d2..1449fa1c7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 53df9fe6` 验证存在 +- 源码校验：以上路径均已在 `main @ 48e4f6a3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1c41b05ab..b1e96ebea 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 53df9fe6`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 48e4f6a3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b57dc70e` 本轮将 PR #6401 的 distributed VAE 通信隔离要求沉淀到 Wan spatial-shard 规则，并补充来源；这只说明专用 WORLD communicator 与该回归路径的合同，不代表所有 VAE、硬件或并行组合都已获得数值或性能支持。 +本次相对 `53df9fe6` 本轮将 PR #6397 沉淀为 MiniCPM-o 4.5 的 MCPMO-1i，约束 Code2Wav 在无 encoder 的 Flow 上跳过 Dynamo 解包。该规则不表示其他模型或所有后端都可跳过解包，也不改变真实 CosyVoice encoder 路径的既有行为。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b57dc70e \ - --to 53df9fe6 \ + --from 53df9fe6 \ + --to 48e4f6a3 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 6796d1405..fd1242c05 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397"] confidence: high --- @@ -114,6 +114,13 @@ confidence: high - 禁止：把数千 codec token 一次送入固定 RelPos PE；忽略 cache offset 或把非最终短片静默截断；用空音频掩盖窗口溢出。 - 验收：单测覆盖预算计算与切片计划，fake Code2Wav 测试断言每片大小、`last_chunk` 和非空音频；不得从该修复推断通用性能提升。 ^[PR #6346] +## MCPMO-1i — Code2Wav dynamo 解包必须兼容无 encoder 的 Flow + +- 触发：MiniCPM-o 4.5 `BatchedToken2Wav` 初始化需要对 `flow.encoder.forward_chunk` 执行 TorchDynamo 解包，但某些 decoder/estimator-only Flow 没有 `encoder`。 +- 强制：先以 `getattr(self.flow, "encoder", None)` 检查 encoder；仅在 encoder 存在时解包 `forward_chunk`，真实 CosyVoice flow 仍必须保留原有解包行为。 +- 禁止：无条件访问 `self.flow.encoder`，或让 encoder-less 的 NPU Code2Wav 图测试与其他合法 Flow 在构造阶段因 `AttributeError` 失败。 +- 验收：decoder/estimator-only `_Flow` 能成功构造并完成 NPU graph/eager 回归；带 encoder 的 CUDA MiniCPM-o 4.5 路径仍确认 `forward_chunk` 的 Dynamo 包装被正确移除。 ^[PR #6397] + ## MCPMO-2a — registry 使用 4.5 config/version predicate - 触发：pipeline auto-detection 看到通用 `MiniCPMO` architecture。 From 7a1f23737da9b61c142c9274ca6ef4c2d789d369 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:20:53 +0800 Subject: [PATCH 230/465] knowledge(vllm-omni): sync 10906e3a (PR #5452) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 10906e3a6db947290581b88dc4b6f35bd2a26b41 Upstream-PR: #5452 Knowledge-Delta: 新增 PR #5452 的多请求 payload/共享状态、generation batch 与 zero-token、sparse 音频协议、采样 fallback/padding 和 capture/replay 一致性规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-bridge-batch.md | 16 +++++++++++++++- .../model-executor/rules-runtime-hot-paths.md | 17 ++++++++++++++++- .../components/model-executor/rules.md | 9 ++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 12 files changed, 52 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e5f31f0dc..49974b001 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 53df9fe62b3e05b388916351ecd758cb770e5cb4 + previous_audited_sha: 48e4f6a38504bc236a28d06e7c2837594c64e46f audited_ref: main - audited_sha: 48e4f6a38504bc236a28d06e7c2837594c64e46f + audited_sha: 10906e3a6db947290581b88dc4b6f35bd2a26b41 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5e1b7d2b4..6323d9f16 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -48e4f6a38504bc236a28d06e7c2837594c64e46f`. The canonical machine baseline +10906e3a6db947290581b88dc4b6f35bd2a26b41`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a96c30af5..3febfcba9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 48e4f6a3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 10906e3a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 105b9a45f..2db5a0659 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 48e4f6a3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 10906e3a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 911fabf7a..e00abacbe 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 48e4f6a3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 10906e3a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8db288279..543f59888 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 48e4f6a3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 10906e3a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md index d31e0cf3c..2cb42e9e0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666", "PR #5491", "PR #6186"] +sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666", "PR #5491", "PR #6186", "PR #5452", "vllm_omni/worker/output/payload_build.py"] confidence: high --- @@ -139,3 +139,17 @@ Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x - 禁止：用 row adjacency 或 internal UUID 推断 pair；让 prefix-cache 不对称命中、chunked prefill 或单边 preemption 使两行错位；只结束一方、把 per-request seed/position metadata 广播给整批，或让非 CFG stage 被无条件改写。 - 验收：覆盖 suffix/显式 pair id、companion 缺失与到达、row reorder/compaction、chunked prefill、prefix-cache control、一起 finish、非 CFG no-op 和 patch 安装时序；同批不同 seed 与位置必须保持各自归属。 ^[PR #6186] +## EXEC-1n — 多请求 payload 与共享调度状态必须保持 request ownership + +- 触发：修改 AR/generation runner 的 multimodal payload 路由、prefix-cache 合并、sparse `meta.req_id`，或模型 KV-transfer metadata 与 downstream payload memoization。 +- 强制：先由共享 resolver 校验 nested/flattened sparse declaration，按 `meta.req_id` 建立 sparse index；payload builder 按 request id/index 取值并过滤协议 metadata，singleton list 只作 request-invariant broadcast；合并模型 KV metadata 必须 copy-on-write，`omni_final_stage_id` 未到达前只能保守返回且不得缓存，获取失败原样抛出。 +- 禁止：用 batch index 代替 sparse index、用 `v[0]` 替代越界多元素值、把请求数据广播给另一 request、原地修改 scheduler-owned dict，或吞掉 transfer metadata 异常。 +- 验收：mixed batch 覆盖 sparse subset、combined prefix-cache/direct path、singleton broadcast、越界丢 key、nested/flat conflict、scheduler 原对象不变、marker 到达后 memoization 更新及异常传播；断言每个 request 只收到自己的 payload。 ^[PR #5452] + +## EXEC-1o — generation runner 必须逐请求构造 batch 输出并保持 zero-token 控制流 + +- 触发：generation runner 支持并发请求，或修改 `sample_tokens`、`total_num_scheduled_tokens <= 0`、encoder transfer、DP dummy-run 与 KV no-forward 分支。 +- 强制：tensor 的 leading dimension 与 list 长度必须等于 `input_batch.num_reqs`，逐请求构造 payload 并在 mismatch 时报告两侧长度；EC producer 分支必须先于 zero-token 分支，zero/negative-token step 仍按条件执行 DP `_dummy_run(1)` 或 `kv_connector_no_forward`。 +- 禁止：用同时要求输出维度为 1 和 `num_reqs` 的断言限制 batch；构造只含一个 entry 的 payload；用无条件 zero-token early return 遮蔽同步、DP 或 KV-transfer 分支。 +- 验收：覆盖两个及以上 request 的 tensor/list 输出、两类长度 mismatch、zero/negative-token、external-launcher + DP、KV no-forward 与 EC producer，分别断言 request 对齐、错误信息和分支调用顺序。 ^[PR #5452] + diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md index cbd4184ea..18f438726 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/] +sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py"] confidence: high --- @@ -41,3 +41,18 @@ confidence: high - 验收：CPU codec stand-in 覆盖跨 first/steady chunk 边界的完整帧拼接、lookback 和 partial final chunk；真实 runner 覆盖 mixed requests、sparse routing、预算尾部与 cleanup；GPU smoke 断言 waveform 样本数等于 committed frames 的实际 frame ledger。 ^[PR #5666] 相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 + +## EXEC-11c — sparse 音频 marker 必须执行协议校验并在非法声明时 fail closed + +- 触发：修改共享 AR 音频输出、sparse multimodal routing、`meta.sparse_audio`/`meta.req_id` producer 或 consumer。 +- 强制：marker 只接受 list of strings 或 bare string，并用同一 literal truthy set 判定；nested 与 flattened encoding 必须同时解析，冲突、重复 id、不可用 id 或非法 carrier 必须记录并在 audio 路由 fail closed；任何 subset producer（包括 VoxCPM2 dense 分支）都必须声明按 `meta.req_id` 对齐。 +- 禁止：对 tensor/ndarray/number marker 做元素读取或归一化、把非法 sparse declaration 降级为 dense、静默抹掉 nested marker，或发送未声明 alignment 的 subset 音频列表。 +- 验收：覆盖 legal list/string、invalid audio/non-audio、unusable `req_id`、nested marker without id、nested/flat conflict、duplicate id、合法 sparse routing 与 VoxCPM2 coalesce/plain producer；并验证一次性 classifier error、rate-limited fail-closed error 和无 D2H 读取。 ^[PR #5452] + +## EXEC-11d — model sampler fallback 与 penalty padding 必须保持显式合同 + +- 触发：模型声明 `prefer_model_sampler`，修改 runner 的 model sampler fallback，或调整 prompt ids 与 logits vocab 的 penalty 处理。 +- 强制：`model.sample()` 返回非 `None` 时直接采用；返回 `None` 必须回退默认 sampler 并通过 `warning_once` 保持可见；prompt padding 必须 clamp 到 `logits_vocab`，保留 upstream 的 padding bin 语义；新增 declarer 必须经过精确 assignment inventory 同意。 +- 禁止：把合法的 `None` fallback 当作采样失败、无日志静默吞掉意外 fallthrough，或 clamp 到 `logits_vocab - 1` 使 padding 被计作最后一个真实 token。 +- 验收：覆盖真实 declarer 的 None/非 None sampler、重复调用仅一次 warning、inventory 增删、padding 对 penalty mask 无影响，以及较窄 logits vocab 下未 clamp 会失败的边界。 ^[PR #5452] + diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 9d5de56be..9f7c73829 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452"] --- # Model Executor 规则 @@ -176,3 +176,10 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 强制：可复用实现以 `vllm_omni/model_executor/models/common/ming/` 为唯一归属，`ming_tts` 与 `ming_flash_omni` 从该目录导入；保留 checkpoint-facing 的类和参数语义，模型侧必须显式传入真实 latent width，且 checkpoint 配置覆盖默认值。 - 禁止：在任一 Ming 模型目录保留近似副本，让一个模型跨包导入另一个模型的可复用实现，或在重命名后遗留旧 import；不能依赖 shared `Aggregator` 的默认 `in_channels` 代替模型真实的 `latent_dim`。 - 验收：共享模块测试断言 `Aggregator`、CFM、speaker extractor 等符号的 `__module__`，确认模型包没有本地重复类；用 Ming-flash 的真实 latent width 构造并断言输出 shape，同时运行 Ming-TTS 与 Ming-flash 的模块回归。 ^[PR #6119] + +- **EXEC-0c — generation capture 与 replay 必须保持 preprocess 输入路径一致** + - 触发：修改 generation runner 的 `_dummy_run`、CUDA-graph capture/replay 输入路径，或模型声明 `has_preprocess`。 + - 强制：generation override 必须在与 base runner 相同的决策点保留 `has_preprocess` 分支；capture 与 replay 必须读取和写入同一对固定 `input_ids.gpu`/`inputs_embeds.gpu` buffer，且不得依赖继承实现的偶然覆盖。 + - 禁止：capture 读取 `input_ids` 而 replay 由 `_preprocess` 写入 `inputs_embeds`，删除 generation override 后让测试仅因 fallback 到 base 实现而通过，或在不同分支改变 graph 输入语义。 + - 验收：静态 AST contract 同时检查 base `_dummy_run`、base `_preprocess` 和 generation `_dummy_run`，并验证 generation override 实际存在；对声明 `has_preprocess` 的 capture/replay case 断言输入 buffer 与输出行为一致。 ^[PR #5452] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 65ff76f34..759fb5e8b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 48e4f6a3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 10906e3a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1449fa1c7..f991f01a9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 48e4f6a3` 验证存在 +- 源码校验：以上路径均已在 `main @ 10906e3a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b1e96ebea..518af136a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 48e4f6a3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 10906e3a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `53df9fe6` 本轮将 PR #6397 沉淀为 MiniCPM-o 4.5 的 MCPMO-1i，约束 Code2Wav 在无 encoder 的 Flow 上跳过 Dynamo 解包。该规则不表示其他模型或所有后端都可跳过解包，也不改变真实 CosyVoice encoder 路径的既有行为。 +本次相对 `48e4f6a3` 本轮沉淀了 PR #5452 的共享 runner 合同：多请求 payload 与共享状态所有权、generation batch/zero-token 路径、sparse-audio marker、采样 fallback/clamp，以及 capture/replay 输入一致性。测试、docs/design 和 VoxCPM2 独立模型页未新增规则；这些规则不能推断所有平台 runner 或所有模型都已验证一致。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 53df9fe6 \ - --to 48e4f6a3 \ + --from 48e4f6a3 \ + --to 10906e3a \ --repo \ --mode report-only ``` From 867a98ed351c2335111a4e37acc42e3b0797acc2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:23:27 +0800 Subject: [PATCH 231/465] knowledge(vllm-omni): treat .markdownlint.yaml as repository tooling Scheduled halt at unit 220, predicted by preflight before the campaign started rather than met at 4am: PR #6273 adds a markdownlint config to the repository root, and the audit reports any root path it cannot route as unmatched_path. The file is a lint rule set -- MD007 indent, MD013 off, MD041 off for READMEs that open with a centered logo. It configures a pre-commit hook and says nothing about inference behaviour, so it belongs beside .pre-commit-config.yaml in ignored_paths rather than routing to a knowledge owner. Committed on its own rather than folded into unit 220. The driver stages `git add -A -- knowledge adapters doc`, so leaving it uncommitted would have merged a policy decision into a sync commit and hidden it from the one place the branch records policy changes. Rerunning preflight drops 220 from the schedule; 232, 259 and 294 remain. No unit is consumed and audited_sha is untouched. Gates: tree 0 errors / 16 reminders, release audit CLEAN. Co-Authored-By: Claude Opus 5 (1M context) --- adapters/vllm_omni/release_baseline.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 49974b001..cb3322a44 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -114,6 +114,8 @@ owner_documents: ignored_paths: - pattern: .gitignore reason: repository metadata +- pattern: .markdownlint.yaml + reason: repository tooling（PR #6273 引入 markdownlint） - pattern: .pre-commit-config.yaml reason: repository tooling - pattern: .readthedocs.yaml From f55c3b416646490ed9b002fe9cdeebbef961de66 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:25:29 +0800 Subject: [PATCH 232/465] knowledge(vllm-omni): sync 273ffce1 (PR #6273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 273ffce130ec648fb0b600d7a6895cfb9c91bd52 Upstream-PR: #6273 Knowledge-Delta: 为 diffusion Triton kernel 记录统一经 `vllm.triton_utils` 导入的兼容性门禁；其余 CI、文档、工具、拼写与测试改动不沉淀。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-upstream-compat.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cb3322a44..1c1533e9b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 48e4f6a38504bc236a28d06e7c2837594c64e46f + previous_audited_sha: 10906e3a6db947290581b88dc4b6f35bd2a26b41 audited_ref: main - audited_sha: 10906e3a6db947290581b88dc4b6f35bd2a26b41 + audited_sha: 273ffce130ec648fb0b600d7a6895cfb9c91bd52 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6323d9f16..590ef67ab 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -10906e3a6db947290581b88dc4b6f35bd2a26b41`. The canonical machine baseline +273ffce130ec648fb0b600d7a6895cfb9c91bd52`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3febfcba9..21922869f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 10906e3a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 273ffce1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2db5a0659..051aaa69e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 10906e3a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 273ffce1` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md index 3cf2f5c8b..f9fba64a8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py, "PR #6287"] +sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py, "PR #6287", "PR #6273"] confidence: high --- @@ -39,3 +39,10 @@ confidence: high - 禁止：假设导入兼容包的子模块不会执行父包初始化；在 ROCm platform patch 已进入 diffusion registry 时重新引入该兼容导入；把此修复泛化为改变 `LoRAConfig` 类型或配置语义。 - 验收：ROCm 启动/pytest collection 在 diffusion 模块完成初始化前不再出现 circular-import；断言两个导入路径解析为同一 `LoRAConfig` 类，并回归 LoRA manager/utils 的正常导入及 CUDA/XPU 初始化路径。^[PR #6287] +## DIFF-4s — diffusion Triton kernel 必须通过 vLLM 兼容入口导入 + +- 触发：修改 diffusion Triton kernel 或其第三方 Triton 导入路径。 +- 强制：diffusion 代码统一从 `vllm.triton_utils` 导入 `triton` 与 `tl`，并保持现有 kernel 装饰器和调用语义。 +- 禁止：直接从 `triton` 或 `triton.language` 导入，或把导入检查通过误认为真实 kernel 已完成跨平台验证。 +- 验收：静态检查确认无直接 Triton 导入，并在支持的 CUDA 环境执行目标 kernel 的导入与定向回归；非 CUDA fallback 仍需独立验证。^[PR #6273] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e00abacbe..d4bf19e9b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 10906e3a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 273ffce1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 543f59888..39905a4c0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 10906e3a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 273ffce1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 759fb5e8b..4a38b3a9c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 10906e3a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 273ffce1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f991f01a9..c7ae5723c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 10906e3a` 验证存在 +- 源码校验：以上路径均已在 `main @ 273ffce1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 518af136a..5ee930773 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 10906e3a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 273ffce1`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `48e4f6a3` 本轮沉淀了 PR #5452 的共享 runner 合同：多请求 payload 与共享状态所有权、generation batch/zero-token 路径、sparse-audio marker、采样 fallback/clamp，以及 capture/replay 输入一致性。测试、docs/design 和 VoxCPM2 独立模型页未新增规则；这些规则不能推断所有平台 runner 或所有模型都已验证一致。 +本次相对 `10906e3a` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 48e4f6a3 \ - --to 10906e3a \ + --from 10906e3a \ + --to 273ffce1 \ --repo \ --mode report-only ``` From 4869f104336700a0249fe41410a31e679bf90a4d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:26:52 +0800 Subject: [PATCH 233/465] knowledge(vllm-omni): sync 718d9a4c (PR #5999) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 718d9a4cd7505f0a7e11d8a1de9d177bb01c556f Upstream-PR: #5999 Knowledge-Delta: Serving 新增图像生成与编辑端点透传 diffusion 阶段耗时和峰值显存指标的审查规则 SERV-2d。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1c1533e9b..2ab1e36b7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 10906e3a6db947290581b88dc4b6f35bd2a26b41 + previous_audited_sha: 273ffce130ec648fb0b600d7a6895cfb9c91bd52 audited_ref: main - audited_sha: 273ffce130ec648fb0b600d7a6895cfb9c91bd52 + audited_sha: 718d9a4cd7505f0a7e11d8a1de9d177bb01c556f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 590ef67ab..fbd42e5c0 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -273ffce130ec648fb0b600d7a6895cfb9c91bd52`. The canonical machine baseline +718d9a4cd7505f0a7e11d8a1de9d177bb01c556f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 21922869f..796a518a0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 273ffce1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 718d9a4c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 051aaa69e..aee5ab255 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 273ffce1` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 718d9a4c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d4bf19e9b..a9b556782 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 273ffce1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 718d9a4c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 39905a4c0..b9b8ae7fa 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 273ffce1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 718d9a4c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4a38b3a9c..541c5d810 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 273ffce1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 718d9a4c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c7ae5723c..741032174 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 273ffce1` 验证存在 +- 源码校验：以上路径均已在 `main @ 718d9a4c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 573950bdf..c0fb1dfbd 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999"] confidence: high --- @@ -172,6 +172,13 @@ confidence: high - 禁止：在轮询重试、重复回调或消息标记后再次累计同一 result；把同一消息误当作新的 denoising step，或绕过现有 consumed 生命周期清理。 - 验收：同一 result 重复处理只产生一次累计，两个 distinct result 各自产生一次；覆盖异常、重试与请求清理，确认 consumed 状态不会导致重复累计或遗留。^[PR #4755] +## SERV-2d — 图像生成与编辑响应必须透传 diffusion 指标 + +- 触发：修改 `/v1/images/generations` 或 `/v1/images/edits` 的 diffusion generation result 解包、指标字段或 OpenAI 响应构造。 +- 强制：单 stage 从 result 属性、multi-stage 从返回 tuple 捕获 `stage_durations` 与 `peak_memory_mb`，并在两个图像端点的响应 `metrics` 中透传；指标不可用时保留 `null`，内存值按稳定的数值类型输出。 +- 禁止：让 `edit_images()` 丢弃已生成的阶段耗时或峰值内存；只修复 generations 路径，或以 HTTP 成功代替指标字段的传播证明。 +- 验收：对 generations 和 edits 分别覆盖 single-stage 与 multi-stage stub 结果，断言响应状态为 200，且 `metrics.stage_durations`、`metrics.peak_memory_mb` 与 generation result 精确一致；同时覆盖指标缺失时的 `null` 响应。^[PR #5999] + ## 缓存 readiness 与失败隔离 ### SERV-3a — readiness 必须表达 artifact 能力而非仅表达 identity diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5ee930773..39f9fd1e9 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 273ffce1`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 718d9a4c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `10906e3a` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `273ffce1` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 10906e3a \ - --to 273ffce1 \ + --from 273ffce1 \ + --to 718d9a4c \ --repo \ --mode report-only ``` From 03b1be829990d708fd08e9097b2479e4bf14815d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:31:15 +0800 Subject: [PATCH 234/465] knowledge(vllm-omni): sync cd3867c3 (PR #5048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: cd3867c373282c5c1dfda9b2f27d44ee33759b1b Upstream-PR: #5048 Knowledge-Delta: PR #5048 新增 `silence_ban_frames` 的 typed 配置透传、GPU/NPU req_id 对齐与 Qwen3-TTS x-vector-only silence mask 合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../model-executor/rules-platform-backends.md | 9 ++++++++- .../model-executor/rules-runtime-hot-paths.md | 9 ++++++++- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/qwen3-tts/rules.md | 11 +++++++++-- 13 files changed, 46 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2ab1e36b7..b52ad885a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 273ffce130ec648fb0b600d7a6895cfb9c91bd52 + previous_audited_sha: 718d9a4cd7505f0a7e11d8a1de9d177bb01c556f audited_ref: main - audited_sha: 718d9a4cd7505f0a7e11d8a1de9d177bb01c556f + audited_sha: cd3867c373282c5c1dfda9b2f27d44ee33759b1b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fbd42e5c0..84d247d8a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -718d9a4cd7505f0a7e11d8a1de9d177bb01c556f`. The canonical machine baseline +cd3867c373282c5c1dfda9b2f27d44ee33759b1b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 796a518a0..8d31743ff 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 718d9a4c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ cd3867c3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index b4f25e982..95f75341c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048"] --- # vLLM-Omni 配置开发门禁 @@ -104,6 +104,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：用字段同名替代语义与 owner 审计；让 positional constructor 因 upstream 字段重排而静默改变含义；把继承字段的默认值当作显式 engine input；在 head process 执行终态初始化，或让 quantization 等未审计边界随 upstream contract 自动扩张。 - 验收：覆盖 keyword-only 签名与 positional rejection、mapping 到具体 upstream object、预构造 object 的类型保留、scheduler/parallel 传输安全 derived fields 及 msgpack/asdict registration；用非默认字段证明最终 EngineArgs projection，且默认值不泄漏、未归属字段明确失败，legacy 与 structured 结果保持一致。 ^[PR #6050] +### VOMNI-CFG-1m — 模型专用 stage 开关必须经 typed projection 且默认关闭 + +- 触发：为某一模型新增控制推理行为的 stage/deploy 字段，并让 deploy、CLI、structured 或 direct factory 可设置。 +- 强制：把字段放在 `StageDeployConfig`、`OmniEngineArgs`、`OmniModelConfig`/`OmniStageModelConfig` 与 `_ModelEngineOverrides` 的同一 canonical projection；默认关闭（本例 `silence_ban_frames=0`），从最终 stage config 传到唯一模型 consumer，并同步更新 structured/legacy known-field inventory。 +- 禁止：用未注册 env var 或模型初始化时自行读取绕过配置 owner；只在 YAML/dataclass 声明但在 engine args 或最终 stage 丢失；把模型专用字段变成所有 talker 的通用行为，或把 absent 当成显式启用。 +- 验收：用 direct、structured、legacy 入口的非默认值追到最终 stage config 和真实模型 consumer；断言默认值保持关闭、其他模型不产生副作用，并覆盖字段集合与 projection parity。^[PR #5048] + ### CONF-1a — 多 stage 共卡时 diffusion stage 必须显式设 gpu_memory_utilization - 触发：多 stage 模型 stage 共卡时，分布式测试在模型加载期 CUDA OOM。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index aee5ab255..5ed8f2426 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 718d9a4c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ cd3867c3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a9b556782..52b38fcb0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 718d9a4c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ cd3867c3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b9b8ae7fa..1e5c99d7a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 718d9a4c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ cd3867c3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index 7d35cb1ba..06c6acc2f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py"] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048"] confidence: high --- @@ -33,6 +33,13 @@ confidence: high - 禁止：为适配 XPU 修改共享 `gpu_ar_model_runner.py`，或只映射 stream 而遗漏 `set_stream`、`set_device`、event 与受支持的 graph 接口；不得让通用 `torch.Stream.wait_event()` 接收不兼容的专用事件类型，也不得在不支持 XPU graph 时无条件安装 graph shim。 - 验收：XPU 平台测试进入 wrapper 后断言异步 runner 所需的 stream、event、device 和 graph 调用均路由到对应 XPU API，且不支持 graph 时保持禁用；Qwen3-TTS 等 AR-stage warm-up 与完整异步输出 smoke 在 XPU 上成功完成，同时 CUDA 路径和共享 runner 行为保持不变。^[PR #5569] +## EXEC-10d — GPU/NPU AR runner 必须镜像逐请求 logits metadata handoff + +- 触发：AR 模型在 logits 计算中需要 runner 提供按 request 的 req_id 或其他逐请求 metadata，且 GPU 与 NPU 有独立的 execute/logits 路径。 +- 强制：GPU/NPU runner 都必须在 `flush_pending_metadata` 后，以 logits 行顺序调用可选的 `set_batch_req_ids(req_ids[:num_reqs])`；仅在 `spec_decode_metadata is None` 时建立一行对应一个 request 的映射，并允许没有该 hook 的模型无操作通过。 +- 禁止：只在 GPU 路径接入、在 metadata flush 前调用、把 padding/dummy request 传入 batch，或在 spec decode 的多行 request 映射下继续按位置记录 req_id。 +- 验收：GPU 与 NPU 的测试分别断言 hook 的调用顺序、参数和 `num_reqs` 截断；覆盖无 hook 模型、mixed batch 与 spec decode，证明错误映射不会进入模型 logits mask。^[PR #5048] + ## EXEC-12a — ROCm 分页注意力必须走 packed KV 的兼容 varlen 路径 - 触发：修改 `vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py` 的 ROCm 分页注意力、AITER/`flash_attn` varlen kernel、`block_table` 映射或 KV block layout。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md index 18f438726..6be935374 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py"] +sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048"] confidence: high --- @@ -56,3 +56,10 @@ confidence: high - 禁止：把合法的 `None` fallback 当作采样失败、无日志静默吞掉意外 fallthrough，或 clamp 到 `logits_vocab - 1` 使 padding 被计作最后一个真实 token。 - 验收：覆盖真实 declarer 的 None/非 None sampler、重复调用仅一次 warning、inventory 增删、padding 对 penalty mask 无影响，以及较窄 logits vocab 下未 clamp 会失败的边界。 ^[PR #5452] +## EXEC-11e — 逐请求 logits mask 必须由 live history 与 batch identity 守住窗口 + +- 触发：`compute_logits` 或 logits processor 按 request 的 sampling metadata 对前 N 个 decode step 施加 mask、penalty 或其他逐请求覆盖。 +- 强制：启用行为时请求并保留 `output_token_ids`；decode step 必须取每个 request 的 live history 长度，窗口严格使用 `step < N`，并按同一 batch 顺序结合 request mode；history 不得由独立 counter 代替，以保证 preemption/resume 不会重新触发窗口。metadata 或 req_id 长度不匹配、未分类 mode 和无法建立一行一请求映射时必须 warning 并 fail closed；没有早期目标行时跳过 device mask 分配。 +- 禁止：把 mask 应用于整个 batch、假定普通采样总会填充 history、用本地 counter 在恢复后重启 ban，或在 spec decode 的多行映射下按位置读取 mode；不得静默吞掉形状不一致而继续按错误 request masking。 +- 验收：覆盖默认关闭、step `0..N-1`、`step >= N`、mixed-step/mixed-mode batch、ICL/未记录 mode、缺失或空 `output_token_ids`、history/req_id mismatch、warning 去重与 preemption history 保留；确认无早期目标时不产生热路径 device allocation。^[PR #5048] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 541c5d810..de3181e9e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 718d9a4c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ cd3867c3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 741032174..868a89d6b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 718d9a4c` 验证存在 +- 源码校验：以上路径均已在 `main @ cd3867c3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 39f9fd1e9..42fbefadf 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 718d9a4c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ cd3867c3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `273ffce1` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `718d9a4c` 本轮沉淀 PR #5048 的 typed 配置透传、GPU/NPU AR runner 的 req_id 对齐、按 output history 的 logits 热路径，以及 Qwen3-TTS 的 checkpoint-derived silence mask 与 x-vector-only 门控。测试文件、手工 H200 结果和单纯的 serving 参数透传未另立规则；不应据此推断其他 checkpoint 已完成验证、NPU 已有硬件 E2E、`N=3` 是通用最优值，或所有 TTS 模式都会启用该 ban。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 273ffce1 \ - --to 718d9a4c \ + --from 718d9a4c \ + --to cd3867c3 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index 2ccedc1df..71d68297f 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -1,10 +1,10 @@ --- title: "Qwen3-TTS 规则" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py] +sources: ["PR #5157", "PR #5202", "PR #5608", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] confidence: high --- @@ -147,3 +147,10 @@ confidence: high 共享 readiness/错误隔离规则见 [Serving rules](../../components/serving/rules.md)； Qwen 家族入口见 [Qwen-Omni](../qwen-omni/_index.md)。 + +## Q3TTS-4a — silence codec ban 必须按 checkpoint 词表和 x-vector-only mode 生效 + +- 触发：修改 Qwen3-TTS Base voice-clone 的 leading silence 行为、`silence_ban_frames`、talker codec mask 或 x-vector/ICL mode resolution。 +- 强制：默认 `silence_ban_frames=0`；加载 checkpoint 后通过 `_encode_ref_audio_batch` 编码多种静音样本并只收集 codebook-0 ids，词表为空、越界或明显过大时记录 warning、清空启用值并禁用功能。decode 时只对 Base 的 x-vector-only request 在前 N 个 history steps 屏蔽派生 ids；mode resolution 必须与 prompt builder 一致，`voice_clone_prompt.icl_mode` 覆盖和未记录 mode 都要正确处理。 +- 禁止：硬编码当前 checkpoint 的 12 个 token、把 mask 施加到 ICL 或非 Base 请求、对不可信派生词表只做部分 masking，或把经验值 `N=3` 当成跨 checkpoint 的固定最优参数。 +- 验收：覆盖默认关闭、有效派生、空/越界/ oversized 派生和 encoder failure；覆盖 step 边界、mixed x-vector/ICL batch、ICL 与非目标请求保持未修改，以及实际 stage 配置能到达 talker。^[PR #5048] From 0280fc8a97e5c8efca6c628f77f0808974d29f91 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:34:35 +0800 Subject: [PATCH 235/465] knowledge(vllm-omni): sync 8d34d3b5 (PR #6306) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 8d34d3b5deed95a3cdfa4ed3b58fdd86ddddca80 Upstream-PR: #6306 Knowledge-Delta: 新增共享 GroupNorm/AdaGN+SiLU 融合算子的数值与布局合同，并记录 HunyuanImage3 平台分派、VAE tiling 与 cuDNN 并发边界。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/rules.md | 8 +++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md | 7 ++++++- 11 files changed, 26 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b52ad885a..c79a0c572 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 718d9a4cd7505f0a7e11d8a1de9d177bb01c556f + previous_audited_sha: cd3867c373282c5c1dfda9b2f27d44ee33759b1b audited_ref: main - audited_sha: cd3867c373282c5c1dfda9b2f27d44ee33759b1b + audited_sha: 8d34d3b5deed95a3cdfa4ed3b58fdd86ddddca80 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 84d247d8a..4cb763802 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -cd3867c373282c5c1dfda9b2f27d44ee33759b1b`. The canonical machine baseline +8d34d3b5deed95a3cdfa4ed3b58fdd86ddddca80`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8d31743ff..b43bd3070 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ cd3867c3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8d34d3b5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5ed8f2426..01c5df2d2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ cd3867c3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8d34d3b5` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 52b38fcb0..b3f0bebd7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ cd3867c3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8d34d3b5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1e5c99d7a..2cfd3aa73 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ cd3867c3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8d34d3b5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 9f7c73829..b4c12a6fb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306"] --- # Model Executor 规则 @@ -183,3 +183,9 @@ Stage 拓扑错误的最小充分源码证据只有三段：一处最终配置 - 禁止：capture 读取 `input_ids` 而 replay 由 `_preprocess` 写入 `inputs_embeds`，删除 generation override 后让测试仅因 fallback 到 base 实现而通过，或在不同分支改变 graph 输入语义。 - 验收：静态 AST contract 同时检查 base `_dummy_run`、base `_preprocess` 和 generation `_dummy_run`，并验证 generation override 实际存在；对声明 `has_preprocess` 的 capture/replay case 断言输入 buffer 与输出行为一致。 ^[PR #5452] +### EXEC-16a — 共享 GroupNorm 融合算子必须保持 eager、数值与布局合同 + +- 触发：在 `vllm_omni/model_executor/models/common/ops/` 增加或复用 `fused_group_norm_silu`、`fused_adaptive_group_norm_silu`，或在混合精度模型中替换 GroupNorm/AdaGN 与 SiLU 组合。 +- 强制：wrapper 必须在 Triton/CUDA 不可用时回退到 eager 实现，支持 `(B, C, *spatial)` 并在计算后恢复原 shape；启动 kernel 前将输入和 `torch.chunk` 产生的 `scale`/`shift` materialize 为所需连续布局；均值和方差使用 FP32 的 Welford/centered reduction，输出 dtype 与 eager GroupNorm 一致，autocast 下保留其 FP32 输出语义。 +- 禁止：仅凭 `HAS_TRITON` 让非 CUDA、unsupported dtype 或错误布局进入 kernel；使用 `E[x^2] - E[x]^2` 计算方差；对 strided conditioning view 使用 `view`；或用同 dtype 的不稳定 reference、输入 dtype 推断 autocast 输出 dtype。 +- 验收：以 eager reference 覆盖 FP32/FP16/BF16、任意 spatial rank、非 contiguous 输入、chunk-derived `scale`/`shift` 和 autocast；加入大偏移小方差用例验证无 NaN 和有界误差，并分别断言 native fallback、shape、dtype 与 CUDA kernel 数值正确性。^[PR #6306] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index de3181e9e..b925efa06 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ cd3867c3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8d34d3b5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 868a89d6b..59369bf1e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ cd3867c3` 验证存在 +- 源码校验：以上路径均已在 `main @ 8d34d3b5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 42fbefadf..aa2853868 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ cd3867c3`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8d34d3b5`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `718d9a4c` 本轮沉淀 PR #5048 的 typed 配置透传、GPU/NPU AR runner 的 req_id 对齐、按 output history 的 logits 热路径，以及 Qwen3-TTS 的 checkpoint-derived silence mask 与 x-vector-only 门控。测试文件、手工 H200 结果和单纯的 serving 参数透传未另立规则；不应据此推断其他 checkpoint 已完成验证、NPU 已有硬件 E2E、`N=3` 是通用最优值，或所有 TTS 模式都会启用该 ban。 +本次相对 `cd3867c3` 本轮记录了 PR #6306 的共享 GroupNorm/AdaGN+SiLU 融合算子合同，以及 HunyuanImage3 的平台分派、VAE tiling 和 cuDNN 边界。其性能提升仅绑定 PR 中的 exact workload，不能推断为跨硬件、跨模型或通用加速结论；测试文件与目录整理未沉淀为独立规则。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 718d9a4c \ - --to cd3867c3 \ + --from cd3867c3 \ + --to 8d34d3b5 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md index b25ee72d8..84522f23d 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-13 updated: 2026-09-04 type: rule tags: [vllm-omni, models, hunyuan-image3] -sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py"] +sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", "PR #6306"] --- # HunyuanImage3 开发规则 @@ -136,3 +136,8 @@ sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/dif - 禁止：保持 cache manager 为普通对象而期待 Worker spec discovery 找到嵌套 attention；把 paged KV 标记省略为默认 dense；只用一张参考图或关闭 CFG 生成 profile 后宣称容量覆盖最大请求；把 Scheduler-only `DiffusionKVRequest` 状态发送到 profile Worker execution。 - 验收：模型测试断言 manager 是 `nn.Module`、attention identity 可从 `named_modules()` 读回且 `state_dict()` 为空；spec discovery 断言 layer、role、BF16 dtype 与 layout capability；profile 测试断言最大参考图数量、profile envelope 和 dense mode 的旁路行为。 ^[PR #6094] +- **HY3-8a — HunyuanImage3 硬件优化层必须保持平台回退、checkpoint 与 VAE 内存边界** + - 触发：修改 HunyuanImage3 的 `layers/`、VAE/DiT `ResnetBlock`/`ResBlock`、patch embedding 相关块、平台分派、VAE 高分辨率内存策略或 cuDNN autotune。 + - 强制：由 `current_omni_platform.is_cuda()` 选择 `nvidia` 实现，其他后端保留 `native` 实现；两套 block 必须保持相同的公开构造与 checkpoint `state_dict` key 布局，NVIDIA 版本只能通过共享融合算子获得加速；高分辨率 VAE 必须由统一 VAE tiling 管理，任何允许 tiling 关闭的入口都要显式定义容量、回退或拒绝策略；`torch.backends.cudnn` 这类进程级标志应在 worker 边界配置，若保留 block 内临时设置则必须证明 forward 不会并发。 + - 禁止：在非 CUDA 平台导入或强制执行 NVIDIA block；把 Triton 可导入等同于所有平台支持；改变 `in_layers`/`out_layers` 等 checkpoint-facing module key；移除逐卷积内存保护后仍让 tiling-disabled 的高分辨率请求无条件执行；或在可能重叠的 forward 中修改进程级 cuDNN 标志。 + - 验收：分别以 native/eager 与 CUDA/NVIDIA 实现覆盖 VAE、DiT 的真实 shape、dtype 和输出误差；验证旧 checkpoint 的 `state_dict` 加载与 key parity；对启用 tiling、关闭 tiling及直接 VAE 入口执行高分辨率容量/失败行为测试，并在并发 worker 或明确串行证明下验证 cuDNN 配置与恢复；性能数字只在固定模型、硬件、并行拓扑和 workload 下复测。^[PR #6306] From 784c268c5764923ae0158e8b8caacd9725bc5b98 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:35:52 +0800 Subject: [PATCH 236/465] knowledge(vllm-omni): sync b1da47b7 (PR #6350) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b1da47b781c02613393f319cab105e26a7d4c374 Upstream-PR: #6350 Knowledge-Delta: 新增 NPU diffusion fused MoE OOT custom op 的生产路径注册、幂等性与最小注册范围合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-platform-backends.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c79a0c572..92cc0dcc6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: cd3867c373282c5c1dfda9b2f27d44ee33759b1b + previous_audited_sha: 8d34d3b5deed95a3cdfa4ed3b58fdd86ddddca80 audited_ref: main - audited_sha: 8d34d3b5deed95a3cdfa4ed3b58fdd86ddddca80 + audited_sha: b1da47b781c02613393f319cab105e26a7d4c374 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4cb763802..1c9197434 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8d34d3b5deed95a3cdfa4ed3b58fdd86ddddca80`. The canonical machine baseline +b1da47b781c02613393f319cab105e26a7d4c374`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b43bd3070..2b8aa2c7b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8d34d3b5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b1da47b7` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 01c5df2d2..e1c8e02e5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8d34d3b5` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b1da47b7` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b3f0bebd7..48daf420b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8d34d3b5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b1da47b7` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2cfd3aa73..7d727992e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8d34d3b5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b1da47b7` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index 06c6acc2f..5de367371 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048"] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350"] confidence: high --- @@ -70,3 +70,10 @@ confidence: high - 禁止：在 NPU platform `__init__` 中导入或执行会经 `pipeline_registry` → `DiffusionOutput` 回环的模型 patch；依赖 import side effect；只移动一个 patch 而让同一模型的相关 patch 仍在过早阶段加载；把 NPU 时序修复外推为所有平台的通用行为。 - 验收：静态或子进程测试证明平台构造不会加载 MiniMax-H3 encoder/diffusion data，runtime hook 在 pipeline 加载前调用目标 patch 且保留其他 patch；验证重复初始化不重复 patch，并用真实 NPU consumer 覆盖 patch 生效路径。^[PR #6293] +## EXEC-13d — NPU diffusion fused MoE 必须在真实 runtime 注册 OOT 实现 + +- 触发：NPU diffusion 路径调用 fused MoE，而 vllm-ascend 的 OOT MoE 注册只在其 `NPUWorker.__init__` 中执行，导致 `RoutedExperts` 未注册或无法接收 `n_shared_experts`。 +- 强制：在生产 `prepare_fused_moe_runtime()` 路径中幂等注册 `MoERunner` 与 `RoutedExperts`；注册前检查 `op_registry_oot`，已有名称必须保留其现有实现，并只依赖 diffusion config shim 实际提供的参数。 +- 禁止：依赖 `NPUWorker` 初始化或 import side effect 建立 diffusion 注册；无条件注册完整 `REGISTERED_ASCEND_OPS` 表；重复调用触发注册断言，或让 `PluggableLayer` 回退到不兼容的上游实现。 +- 验收：从真实 NPU diffusion runtime 覆盖首次和重复初始化，断言两个 OOT 名称均解析到可接受 `n_shared_experts` 的实现并完成 HunyuanImage3 fused-MoE accuracy/performance smoke；同时确认普通 NPU、非 NPU 和已完成 vllm-ascend 注册的路径不变。^[PR #6350] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b925efa06..caf6c7570 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8d34d3b5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b1da47b7` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 59369bf1e..5470fc35b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8d34d3b5` 验证存在 +- 源码校验：以上路径均已在 `main @ b1da47b7` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index aa2853868..ff904a973 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8d34d3b5`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b1da47b7`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `cd3867c3` 本轮记录了 PR #6306 的共享 GroupNorm/AdaGN+SiLU 融合算子合同，以及 HunyuanImage3 的平台分派、VAE tiling 和 cuDNN 边界。其性能提升仅绑定 PR 中的 exact workload，不能推断为跨硬件、跨模型或通用加速结论；测试文件与目录整理未沉淀为独立规则。 +本次相对 `8d34d3b5` 本轮新增 EXEC-13d，记录 NPU diffusion 路径在真实 fused MoE runtime 中幂等注册 vllm-ascend 的 OOT `MoERunner`/`RoutedExperts` 实现，并限制注册范围以避免 diffusion 配置 shim 触发其他 Ascend 算子假设。该规则不能推断所有 Ascend custom ops、所有模型或非 NPU 平台都应采用同样注册方式。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from cd3867c3 \ - --to 8d34d3b5 \ + --from 8d34d3b5 \ + --to b1da47b7 \ --repo \ --mode report-only ``` From 7f0866d23c8ff9902298e30b03412577015695ee Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:37:29 +0800 Subject: [PATCH 237/465] knowledge(vllm-omni): sync c6e38860 (PR #5445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: c6e38860e9419b5fe8ea90b262ba9439c5d06df6 Upstream-PR: #5445 Knowledge-Delta: 新增 serving stage-replica-layout 规则：省略 devices 时必须为每个副本生成可索引的 None 设备槽位并完成启动计划。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 92cc0dcc6..4ae1e3a6c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8d34d3b5deed95a3cdfa4ed3b58fdd86ddddca80 + previous_audited_sha: b1da47b781c02613393f319cab105e26a7d4c374 audited_ref: main - audited_sha: b1da47b781c02613393f319cab105e26a7d4c374 + audited_sha: c6e38860e9419b5fe8ea90b262ba9439c5d06df6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1c9197434..8f06a3ea4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b1da47b781c02613393f319cab105e26a7d4c374`. The canonical machine baseline +c6e38860e9419b5fe8ea90b262ba9439c5d06df6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2b8aa2c7b..b311525b1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b1da47b7` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c6e38860` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e1c8e02e5..de5c5f8c5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b1da47b7` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c6e38860` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 48daf420b..d6a931957 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b1da47b7` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c6e38860` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7d727992e..bef075a35 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b1da47b7` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c6e38860` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index caf6c7570..7d87a2192 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b1da47b7` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c6e38860` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5470fc35b..a00a6fd02 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b1da47b7` 验证存在 +- 源码校验：以上路径均已在 `main @ c6e38860` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index c0fb1dfbd..af39214f9 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445"] confidence: high --- @@ -226,3 +226,10 @@ confidence: high - 强制：INFO 级别只记录 request ID、模型类型、voice clone、reference image 和参数等元数据；用户输入仅在 `self.request_logger` 存在时通过 `logger.debug` 输出，并按 `max_log_len` 预留日志前缀长度进行截断；未配置上限时使用 200 字符默认上限。 - 禁止：在 INFO 日志中写入原始 `text` 或 `prompt`；无条件输出 DEBUG 用户内容；使用固定预览长度替代请求日志开关；在未配置 `max_log_len` 时输出无界用户输入。 - 验收：覆盖 TTS、diffusion TTS、audio generation 和 diffusion chat，断言 INFO 不含用户内容；启用 `--enable-log-requests` 时 DEBUG 才输出内容，且配置与未配置 `max_log_len` 时均遵守截断上限。 ^[PR #6329] + +### SERV-12a — 多副本 stage 省略 devices 时仍按 replica 数生成设备槽位 + +- 触发：修改 `split_devices_for_replicas()`、`compute_replica_layout()`、`StageRuntime._build_logical_stage_init_plans()`，或修改 stage 的 `num_replicas` 与可选 `devices` 语义。 +- 强制：当 stage 未声明 `devices` 时，返回与有效 replica 数对应的 `None` 槽位（至少保留一个槽位），使 `replica_devices_map` 和启动计划可按每个 `replica_id` 索引；每个 replica 继承 launcher 的 `CUDA_VISIBLE_DEVICES`，显式设备配置继续按既有规则拆分。 +- 禁止：因 `devices=None` 只返回单元素列表；让多副本 stage 的启动计划索引越界；将未声明设备误当成显式设备，或改变显式设备配置与既有 `ValueError` 语义。 +- 验收：覆盖 1、2、4 副本的未声明设备拆分、三副本 `replica_devices_map` 以及 logical stage init plans 的全部 replica ID 和 `None` 设备值；回归显式设备拆分、模板/池模式及错误路径。^[PR #5445] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ff904a973..f5ce3cf37 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b1da47b7`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c6e38860`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `8d34d3b5` 本轮新增 EXEC-13d，记录 NPU diffusion 路径在真实 fused MoE runtime 中幂等注册 vllm-ascend 的 OOT `MoERunner`/`RoutedExperts` 实现，并限制注册范围以避免 diffusion 配置 shim 触发其他 Ascend 算子假设。该规则不能推断所有 Ascend custom ops、所有模型或非 NPU 平台都应采用同样注册方式。 +本次相对 `b1da47b7` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 8d34d3b5 \ - --to b1da47b7 \ + --from b1da47b7 \ + --to c6e38860 \ --repo \ --mode report-only ``` From 5c59b7ac1210977115e72d2cb02f725ec20de388 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:38:01 +0800 Subject: [PATCH 238/465] knowledge(vllm-omni): sync ebb4abde (PR #6339) Upstream-Commit: ebb4abde0a24b2982d94c70ac1ed5e40ab3caa03 Upstream-PR: #6339 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4ae1e3a6c..7bf3e015c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b1da47b781c02613393f319cab105e26a7d4c374 + previous_audited_sha: c6e38860e9419b5fe8ea90b262ba9439c5d06df6 audited_ref: main - audited_sha: c6e38860e9419b5fe8ea90b262ba9439c5d06df6 + audited_sha: ebb4abde0a24b2982d94c70ac1ed5e40ab3caa03 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8f06a3ea4..2314bef28 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c6e38860e9419b5fe8ea90b262ba9439c5d06df6`. The canonical machine baseline +ebb4abde0a24b2982d94c70ac1ed5e40ab3caa03`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b311525b1..d0a49ac86 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c6e38860` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ebb4abde` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index de5c5f8c5..ed6cbe5d3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c6e38860` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ebb4abde` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d6a931957..49d54e024 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c6e38860` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ebb4abde` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index bef075a35..7c6be0511 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c6e38860` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ebb4abde` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7d87a2192..adba9512b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c6e38860` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ebb4abde` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a00a6fd02..0135ca767 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c6e38860` 验证存在 +- 源码校验：以上路径均已在 `main @ ebb4abde` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f5ce3cf37..4a420dd55 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c6e38860`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ebb4abde`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b1da47b7` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `c6e38860` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b1da47b7 \ - --to c6e38860 \ + --from c6e38860 \ + --to ebb4abde \ --repo \ --mode report-only ``` From 3f55c90125b1a4ce209c63a7729f1653d75aaa29 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:40:55 +0800 Subject: [PATCH 239/465] knowledge(vllm-omni): sync b3270dec (PR #5614) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b3270dec8d758b9801aa0d01bd7205757b65b814 Upstream-PR: #5614 Knowledge-Delta: 新增 Cosmos3 transfer 请求/提示词与 negative metadata 合同、model_extras 参数声明来源，并记录多控制 attention 的独立加权实现及 Ulysses 复制成本边界。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules-attention.md | 8 +++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../model-executor/rules-image-task-envelope.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/cosmos3/rules.md | 9 ++++++++- 12 files changed, 29 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7bf3e015c..089173990 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c6e38860e9419b5fe8ea90b262ba9439c5d06df6 + previous_audited_sha: ebb4abde0a24b2982d94c70ac1ed5e40ab3caa03 audited_ref: main - audited_sha: ebb4abde0a24b2982d94c70ac1ed5e40ab3caa03 + audited_sha: b3270dec8d758b9801aa0d01bd7205757b65b814 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2314bef28..7ef2e3440 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ebb4abde0a24b2982d94c70ac1ed5e40ab3caa03`. The canonical machine baseline +b3270dec8d758b9801aa0d01bd7205757b65b814`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d0a49ac86..8b668aad2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ebb4abde` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b3270dec` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ed6cbe5d3..406d846ae 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ebb4abde` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b3270dec` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index c42ec55d1..f98f3f76a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py"] +sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614"] confidence: high --- @@ -91,3 +91,9 @@ confidence: high - 禁止：把单 token 形状交给 cuDNN 后等待 eager exception；用包含 FLASH/MATH 的 priority list 掩盖 backend 选择；因 Q/K 长度不同而丢弃已有 attention mask。 - 验收：CPU/mock 参数化覆盖 `(1,1)`、`(8,1)`、`(1,8)` 与 `(8,8)`，精确断言选择的 SDP backend；另覆盖 compile、masked unequal-length 与 unmasked native path，并说明真实 GPU 数值仍需独立验证。^[PR #6070] +## DIFF-7a — 多控制 transfer attention 必须按控制独立计算并显式承担 Ulysses 复制成本 + +- 触发：修改 Cosmos3 transformer 的多控制 transfer attention、控制 token packing、控制权重传递或其 Ulysses/sequence-parallel 路径。 +- 强制：多控制 attention 必须为每个 control 独立构造 `[text, control_i, target]` 的 K/V 序列；保留该 control 自己的输出，target 输出按已归一化的 control weights 加权后再拼回控制区与 target 区。control token size 与 weight 数量必须一一对应，所有 control range 必须为正且完整覆盖 packed control 区。该路径使用 `skip_sequence_parallel=True` 的专用 attention，multi-control 时不得执行 `gen_sp_prepare`/`gen_sp_gather`，以完整的 `[control_i | target]` 序列在每个 Ulysses rank 复制执行，并显式告警其不降低每-rank memory 或 latency；未提供权重时使用均匀相对权重。 +- 禁止：把多个 control 直接拼成一个共享 attention 序列后让一个 control 污染其他 control 的输出；按 control token 数对拼接序列做 Ulysses 切分；只验证 output shape 就宣称 sequence parallel 为多控制 transfer 提供 per-rank 节省，或把一个控制的局部测试外推为多控制数值/性能 parity。 +- 验收：CPU/mock 测试用至少两个 control 的不同 value 和非均匀权重断言 target 是加权结果、control 输出来自各自 pass，并覆盖 token-size/weight 长度及非法权重；带 Ulysses 的测试断言专用 attention、复制执行和 warning，真实多卡再与单卡 dense reference 对照并分别测量显存、延迟和输出质量。^[PR #5614] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 49d54e024..99ba30222 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ebb4abde` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b3270dec` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7c6be0511..aa0d50434 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ebb4abde` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b3270dec` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md index f931cfb0d..c1c39f3ec 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #6049", "PR #6076", vllm_omni/model_extras/, "PR #6070", "vllm_omni/model_extras/registry.py"] +sources: ["PR #6049", "PR #6076", vllm_omni/model_extras/, "PR #6070", "vllm_omni/model_extras/registry.py", "PR #5614"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index adba9512b..a3d09426d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ebb4abde` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b3270dec` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0135ca767..8d8bb2fa7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ebb4abde` 验证存在 +- 源码校验：以上路径均已在 `main @ b3270dec` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4a420dd55..0e71de18e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ebb4abde`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b3270dec`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `c6e38860` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `ebb4abde` 本轮沉淀 Cosmos3 transfer 的请求契约与多控制 attention 边界：模型页记录 control weight、提示词/metadata 默认值和开关；共享 diffusion attention 页记录多控制输出加权及 Ulysses 复制限制。单测和示例不能推断跨硬件质量、吞吐或通用多模型支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from c6e38860 \ - --to ebb4abde \ + --from ebb4abde \ + --to b3270dec \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index 448063701..b001143f8 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107"] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614"] confidence: high --- @@ -110,3 +110,10 @@ confidence: high - 强制：在公共依赖中声明 `cosmos-guardrail>=0.3.1`，使标准 vLLM-Omni 镜像具备默认 guardrails 所需包；guardrail import 失败时必须立即抛出明确的 `ValueError`，说明 guardrails 已启用但不可用、缺少 `cosmos-guardrail`，并保留 NVIDIA Open Model License Agreement 合规提示。 - 禁止：依赖用户手动安装额外包才能获得默认行为；缺少 guardrail 包时静默继续、把状态描述为用户主动禁用了 safety checker，或把安装成功外推为 HF authentication、模型质量和其他变体能力已验证。 - 验收：干净环境安装公共 requirements 后 `cosmos-guardrail` 可导入，Cosmos3 默认 guardrails 路径可进入模型运行；模拟缺少该包时启动/构造 fail fast，错误文本同时包含包名和许可证链接。^[PR #6107] + +## COSMOS-5a — Transfer 请求与提示词合同 + +- 触发：修改 Cosmos3 transfer 的 `edge`、`blur`、`depth`、`seg`、`wsm` 控制提示解析、控制权重或 transfer prompt 构造。 +- 强制：每个 active hint 只接受声明的字段，并将 `control_weight` 校验为 finite、non-negative 且总和为正；按 hint 声明顺序归一化为相对权重，单个正权重归一为 `1.0`，单控制的绝对强度仍由 `control_guidance` 控制。Transfer 必须使用专用 system prompt，同时对 CFG 两支启用；默认在 positive prompt 追加列出 active hints 并要求逐帧遵循形状、位置和运动的 directive，`emphasize_control_in_prompt=false` 才关闭。默认启用 duration/FPS 与 resolution metadata，并以 `negative_metadata_mode=same` 同步到 negative branch；可分别关闭模板，模式只允许 `same`、`inverse`、`none`。Transfer 不自动生成 negative prompt，调用方需显式传入可选参考 prompt。 +- 禁止：把 `weight` 等未声明字段静默当作 `control_weight`；接受负数、非 finite 或全零权重；用单控制归一化权重改变绝对 guidance；让 request 的 `system_prompt`/`use_system_prompt` 覆盖 transfer system prompt；把默认 metadata 或 negative prompt 行为外推到普通 T2I/T2V。 +- 验收：覆盖单/多控制权重归一化、未知字段、负数、全零和 `emphasize_control_in_prompt` ablation；断言 positive/negative 两支的 system prompt、metadata mode、模板和显式/省略 negative prompt；真实 serving 还需固定模型 revision、输入、seed、步数和硬件分别验证输出质量，当前单测与附件不构成通用质量或性能证据。^[PR #5614] From dbdc79c1ae9912ceff0c6a56fad95054cc4d5a6e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:41:29 +0800 Subject: [PATCH 240/465] knowledge(vllm-omni): sync b2011a3e (PR #6201) Upstream-Commit: b2011a3edda1dbea8057067957a27559114a15b6 Upstream-PR: #6201 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 089173990..cbcd860c2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ebb4abde0a24b2982d94c70ac1ed5e40ab3caa03 + previous_audited_sha: b3270dec8d758b9801aa0d01bd7205757b65b814 audited_ref: main - audited_sha: b3270dec8d758b9801aa0d01bd7205757b65b814 + audited_sha: b2011a3edda1dbea8057067957a27559114a15b6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7ef2e3440..b3d00e1d6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b3270dec8d758b9801aa0d01bd7205757b65b814`. The canonical machine baseline +b2011a3edda1dbea8057067957a27559114a15b6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8b668aad2..8cc1c1466 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b3270dec` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b2011a3e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 406d846ae..e4aeab3e2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b3270dec` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b2011a3e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 99ba30222..0b5cc0ef9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b3270dec` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b2011a3e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index aa0d50434..9c5bcecbe 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b3270dec` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b2011a3e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a3d09426d..6e6d2a5bf 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b3270dec` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b2011a3e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8d8bb2fa7..fae1000dd 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b3270dec` 验证存在 +- 源码校验：以上路径均已在 `main @ b2011a3e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0e71de18e..f2f2f6a8a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b3270dec`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b2011a3e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `ebb4abde` 本轮沉淀 Cosmos3 transfer 的请求契约与多控制 attention 边界：模型页记录 control weight、提示词/metadata 默认值和开关；共享 diffusion attention 页记录多控制输出加权及 Ulysses 复制限制。单测和示例不能推断跨硬件质量、吞吐或通用多模型支持。 +本次相对 `b3270dec` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from ebb4abde \ - --to b3270dec \ + --from b3270dec \ + --to b2011a3e \ --repo \ --mode report-only ``` From cfc20347cd5e3097e01d6c6da0e287e9bc028962 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:43:20 +0800 Subject: [PATCH 241/465] knowledge(vllm-omni): sync 13c2de68 (PR #6325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 13c2de684c3fca3882e508b7ed5a1add1e405504 Upstream-PR: #6325 Knowledge-Delta: 新增 COSMOS-5b，记录 Cosmos3 transfer canonical bucket、尺寸/比例冲突告警与跨 rank JSON aspect_ratio 元数据一致性。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/cosmos3/rules.md | 10 +++++++++- 10 files changed, 22 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cbcd860c2..4b7c9d0e2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b3270dec8d758b9801aa0d01bd7205757b65b814 + previous_audited_sha: b2011a3edda1dbea8057067957a27559114a15b6 audited_ref: main - audited_sha: b2011a3edda1dbea8057067957a27559114a15b6 + audited_sha: 13c2de684c3fca3882e508b7ed5a1add1e405504 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b3d00e1d6..a71d4b482 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b2011a3edda1dbea8057067957a27559114a15b6`. The canonical machine baseline +13c2de684c3fca3882e508b7ed5a1add1e405504`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8cc1c1466..4e803de96 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b2011a3e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 13c2de68` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e4aeab3e2..b3aacb24a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b2011a3e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 13c2de68` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0b5cc0ef9..4e9347563 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b2011a3e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 13c2de68` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9c5bcecbe..ad5e7d36b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b2011a3e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 13c2de68` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6e6d2a5bf..328511dc1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b2011a3e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 13c2de68` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fae1000dd..fc98cb50e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b2011a3e` 验证存在 +- 源码校验：以上路径均已在 `main @ 13c2de68` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f2f2f6a8a..449b1e525 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b2011a3e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 13c2de68`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b3270dec` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `b2011a3e` 本轮将 PR #6325 的 Cosmos3 transfer canonical bucket、size/aspect_ratio 冲突告警、多 rank JSON prompt 元数据重写沉淀为 COSMOS-5b；测试变更与常量搬移未单独形成规则。该规则不表示任意 diffusion pipeline 支持任意尺寸，也不构成多卡质量或性能 parity 证据。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b3270dec \ - --to b2011a3e \ + --from b2011a3e \ + --to 13c2de68 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index b001143f8..35ab12f92 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614"] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325"] confidence: high --- @@ -117,3 +117,11 @@ confidence: high - 强制：每个 active hint 只接受声明的字段，并将 `control_weight` 校验为 finite、non-negative 且总和为正；按 hint 声明顺序归一化为相对权重，单个正权重归一为 `1.0`，单控制的绝对强度仍由 `control_guidance` 控制。Transfer 必须使用专用 system prompt，同时对 CFG 两支启用；默认在 positive prompt 追加列出 active hints 并要求逐帧遵循形状、位置和运动的 directive，`emphasize_control_in_prompt=false` 才关闭。默认启用 duration/FPS 与 resolution metadata，并以 `negative_metadata_mode=same` 同步到 negative branch；可分别关闭模板，模式只允许 `same`、`inverse`、`none`。Transfer 不自动生成 negative prompt，调用方需显式传入可选参考 prompt。 - 禁止：把 `weight` 等未声明字段静默当作 `control_weight`；接受负数、非 finite 或全零权重；用单控制归一化权重改变绝对 guidance；让 request 的 `system_prompt`/`use_system_prompt` 覆盖 transfer system prompt；把默认 metadata 或 negative prompt 行为外推到普通 T2I/T2V。 - 验收：覆盖单/多控制权重归一化、未知字段、负数、全零和 `emphasize_control_in_prompt` ablation；断言 positive/negative 两支的 system prompt、metadata mode、模板和显式/省略 negative prompt；真实 serving 还需固定模型 revision、输入、seed、步数和硬件分别验证输出质量，当前单测与附件不构成通用质量或性能证据。^[PR #5614] + +## COSMOS-5b — Transfer canonical bucket 与 JSON aspect_ratio 元数据必须一致 + +- 触发：修改 Cosmos3 transfer 的 `size`、`resolution`、`aspect_ratio`、控制输入 bucket 选择、JSON prompt 元数据，或 I2V/V2V 的 canvas ratio 格式化与多 rank prompt 处理。 +- 强制：Transfer 必须根据控制输入和选定 resolution 从 `VIDEO_RES_SIZE_INFO` 选择 canonical `(height, width, aspect_ratio)` bucket；原始请求尺寸先保存到 `COSMOS3_TRANSFER_REQUESTED_SIZE_KEY`，仅用于冲突告警，不能改变控制输入选出的几何。Transfer JSON prompt、以及 I2V/V2V 生成 canvas 的 aspect ratio 元数据必须使用 canonical ratio；若请求或 JSON prompt 的 ratio 与 canvas 冲突，所有 rank 都重写元数据，只有 rank 0 输出 warning。非 canonical canvas 必须映射到最近的受支持 ratio，而不是使用 gcd 形成新标签。 +- 禁止：把 Transfer 的任意 `size=HxW` 当作最终几何或宣称其会被采用；保留控制输入已覆盖的 stale `aspect_ratio`；将元数据重写也放在 `_is_rank_zero()` 条件内导致 rank 间 token ids 不一致；用非 canonical 的精确 gcd ratio 触发无意义的冲突告警。 +- 验收：覆盖 Transfer 请求尺寸与控制 bucket 不一致、请求/JSON ratio 冲突、rank 0 与非 rank 0 的 warning/rewrite 行为，以及 `1104x816`、`832x468` 等近似 canvas 映射到 `4,3`、`16,9` 的回归；JSON prompt 解析后必须逐 rank 得到相同 canonical `aspect_ratio`。^[PR #6325] + From 7dcf8c924d62e19f78b6f356ac47a7c53d23d5d6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:46:07 +0800 Subject: [PATCH 242/465] knowledge(vllm-omni): sync 37c11e6e (PR #5791) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 37c11e6e4f60f9edf19e103d7278391def9caa28 Upstream-PR: #5791 Knowledge-Delta: 新增 fused code predictor 的 source shard 完整性、wrapper 命名归一、TP 限制及 NPU/MOSS consumer 同步合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-loader-contract.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4b7c9d0e2..26759eb0d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b2011a3edda1dbea8057067957a27559114a15b6 + previous_audited_sha: 13c2de684c3fca3882e508b7ed5a1add1e405504 audited_ref: main - audited_sha: 13c2de684c3fca3882e508b7ed5a1add1e405504 + audited_sha: 37c11e6e4f60f9edf19e103d7278391def9caa28 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a71d4b482..bfec68624 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -13c2de684c3fca3882e508b7ed5a1add1e405504`. The canonical machine baseline +37c11e6e4f60f9edf19e103d7278391def9caa28`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4e803de96..7ace6e7f1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 13c2de68` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 37c11e6e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b3aacb24a..0a060a4d9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 13c2de68` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 37c11e6e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4e9347563..8d2ffda4d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 13c2de68` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 37c11e6e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ad5e7d36b..6a9b5fe49 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 13c2de68` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 37c11e6e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md index 57447bd6a..8543cbb98 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119"] +sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py"] confidence: high --- @@ -64,3 +64,10 @@ confidence: high - 禁止：在调用方重新实现相反失败语义，绕过带锁和 ModelScope 支持的下载 helper，缓存未命中时静默返回原始 repo id，或为了单个辅助文件下载整个仓库。 - 验收：覆盖本地目录短路、缓存命中、缓存未命中抛错、下载 helper 路由、`allow_patterns`/`cache_dir` 转发及 ModelScope 解析路径；调用方测试必须证明其选择的失败和下载策略。 ^[PR #6119] +## EXEC-2f — fused code predictor 必须保持 source shard、wrapper 前缀与平台路径闭环 + +- 触发：共享 code predictor 将 q/k/v、gate/up 改为 plain `nn.Linear` fused projection，或修改 Qwen3-TTS、Qwen3-Omni、MOSS consumer、checkpoint loader、TP plan 或 NPU override。 +- 强制：fused layout 必须分别保持 `[q,k,v]` 与 `[gate,up]` 行拼接；`CodePredictorBaseModel.load_weights` 按 layer、weight/bias 和 source shard 缓存后再拼装，并对部分或整组缺失硬失败。wrapper 必须归一化 `model.*`、`layers.*`、`codec_embedding.*`、`norm.*`、`rotary_emb.*` 以及 nested `talker.code_predictor.` 前缀后委托共享 loader，并恢复 returned loaded-name 前缀；MOSS local、Qwen3-Omni 与 NPU override 必须使用共享 loader、`qkv_proj` 和 `_split_qkv`。plain packed `nn.Linear` 在具备 TP-aware packing/loading/split 前保持 TP=1 与空 `base_model_tp_plan`。 +- 禁止：用逐 tensor `default_weight_loader` 绕过 fused assembler；以命中一个 source shard 或 target name 代表整组完整；保留已删除的 `q_proj`/`k_proj`/`v_proj` 属性、让 body-direct 或 nested shards 绕过 wrapper 归一化，或对 packed `[q,k,v]`/`[gate,up]` layout 宣告泛化 colwise TP。 +- 验收：用非等 q/KV width 数值核对 `cat([q,k,v])`、`cat([gate,up])` 及 bias，覆盖部分/整组缺 shard；覆盖 current、body-direct、nested 前缀和跨 shard incremental loading；断言 MOSS returned names、Qwen wrapper consumer、NPU mocked `prepare_qkv_weights`/forward 均走 fused 属性，并确认无 TP plan。 ^[PR #5791] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 328511dc1..6420d6a65 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 13c2de68` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 37c11e6e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fc98cb50e..38539926f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 13c2de68` 验证存在 +- 源码校验：以上路径均已在 `main @ 37c11e6e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 449b1e525..493f79c89 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 13c2de68`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 37c11e6e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `b2011a3e` 本轮将 PR #6325 的 Cosmos3 transfer canonical bucket、size/aspect_ratio 冲突告警、多 rank JSON prompt 元数据重写沉淀为 COSMOS-5b；测试变更与常量搬移未单独形成规则。该规则不表示任意 diffusion pipeline 支持任意尺寸，也不构成多卡质量或性能 parity 证据。 +本次相对 `13c2de68` 本轮将 PR #5791 的 code predictor 融合投影、HF shard 装载、wrapper 命名归一、TP 限制与 NPU consumer 同步合同写入 model-executor loader 规则。此前关于 Qwen3 code predictor 仍使用分离 projection 的描述仅代表旧基线；该 PR 的性能与音质验证也不可外推为通用收益。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b2011a3e \ - --to 13c2de68 \ + --from 13c2de68 \ + --to 37c11e6e \ --repo \ --mode report-only ``` From cd36a75adc05bea55cdead6229fd3cc69f15dd61 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 08:47:44 +0800 Subject: [PATCH 243/465] knowledge(vllm-omni): sync 5730af0a (PR #5181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 5730af0a95299387c233e12dfbdbe728aace0e58 Upstream-PR: #5181 Knowledge-Delta: 新增 SERV-4n，规定 Omni Chat Completions 在启用 prompt token details 时保留零缓存值并传播多模态 token 计数。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-request-input.md | 10 +++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 22 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 26759eb0d..e42ab9b87 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 13c2de684c3fca3882e508b7ed5a1add1e405504 + previous_audited_sha: 37c11e6e4f60f9edf19e103d7278391def9caa28 audited_ref: main - audited_sha: 37c11e6e4f60f9edf19e103d7278391def9caa28 + audited_sha: 5730af0a95299387c233e12dfbdbe728aace0e58 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index bfec68624..fd3998e75 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -37c11e6e4f60f9edf19e103d7278391def9caa28`. The canonical machine baseline +5730af0a95299387c233e12dfbdbe728aace0e58`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7ace6e7f1..1c6f226ff 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 37c11e6e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5730af0a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0a060a4d9..1e765f25d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 37c11e6e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5730af0a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8d2ffda4d..c85738a21 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 37c11e6e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5730af0a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6a9b5fe49..f3b12da35 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 37c11e6e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5730af0a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6420d6a65..6e07e86fd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 37c11e6e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5730af0a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 38539926f..2d76e37da 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 37c11e6e` 验证存在 +- 源码校验：以上路径均已在 `main @ 5730af0a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md index fab682e59..9b6779f53 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3805", "PR #5374", vllm_omni/entrypoints/openai/, vllm_omni/inputs/] +sources: ["PR #3805", "PR #5374", vllm_omni/entrypoints/openai/, vllm_omni/inputs/, "PR #5181"] confidence: high --- @@ -110,3 +110,11 @@ confidence: high - 验收：分别验证 positional model、`--model`、`--gpu_memory_utilization` 等下划线参数和连字符参数均可解析，并断言 `args.model` 与 `args.model_tag` 一致且服务加载请求模型，不会回退到默认 LLM。^[PR #3805] engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)；故障隔离见 [fault isolation 规则](rules-fault-isolation.md)。 + +## SERV-4n — Omni Chat Completions 的 prompt token details 必须保留零值与多模态计数 + +- 触发：修改 Omni Chat Completions 的 streaming/non-streaming usage 序列化、`prompt_tokens_details`、prefix-cache 统计或多模态 token 计数传播。 +- 强制：启用 `enable_prompt_tokens_details` 时，streaming 与 non-streaming 路径都必须复用 upstream 的 prompt-token-details helper，传递 engine prompt 的 multimodal token counts，并保留 `cached_tokens` 为 `0` 的详情对象；未启用时不得为此执行额外计数。 +- 禁止：用 truthiness 判断 `num_cached_tokens` 而丢弃零值；只在一种响应模式填充详情；丢弃 `multimodal_tokens` 或以不同逻辑分别构造两种 usage；将该响应合同混同为 Prometheus 指标合同。 +- 验收：分别覆盖 streaming 与 non-streaming、`cached_tokens=0` 及非零值、image/audio 多模态计数和详情开关关闭场景，断言响应字段一致、计数准确且关闭时不产生详情。^[PR #5181] + diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 493f79c89..6a3b2407c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 37c11e6e`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5730af0a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `13c2de68` 本轮将 PR #5791 的 code predictor 融合投影、HF shard 装载、wrapper 命名归一、TP 限制与 NPU consumer 同步合同写入 model-executor loader 规则。此前关于 Qwen3 code predictor 仍使用分离 projection 的描述仅代表旧基线；该 PR 的性能与音质验证也不可外推为通用收益。 +本次相对 `37c11e6e` 本轮新增 Omni Chat Completions 的 prompt token details 合同：启用详情时，streaming 与 non-streaming 都必须保留 `cached_tokens: 0` 并传播多模态 token 计数。该规则不代表所有 endpoint 或 Prometheus 指标都采用相同字段，也不覆盖模型专属 token 统计语义。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 13c2de68 \ - --to 37c11e6e \ + --from 37c11e6e \ + --to 5730af0a \ --repo \ --mode report-only ``` From 3a8d3240a87aac025e503b6a4f827eb2b8017149 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:12:06 +0800 Subject: [PATCH 244/465] knowledge(vllm-omni): sync 73b623f2 (PR #6419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 73b623f2f7db092053c1c86fe796bed89eb3dc71 Upstream-PR: #6419 Knowledge-Delta: 本轮提交的 Host Weight Runtime 基础规则已由现有 HWR-1a、HWR-1b、HWR-2a 覆盖，未产生新的持久知识。 --- adapters/vllm_omni/release_baseline.yaml | 8 ++++++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 17 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e42ab9b87..81858efc1 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 37c11e6e4f60f9edf19e103d7278391def9caa28 + previous_audited_sha: 5730af0a95299387c233e12dfbdbe728aace0e58 audited_ref: main - audited_sha: 5730af0a95299387c233e12dfbdbe728aace0e58 + audited_sha: 73b623f2f7db092053c1c86fe796bed89eb3dc71 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -81,6 +81,8 @@ path_owners: - vllm_omni/patch.py - vllm_omni/request.py + host-weight-runtime: + - vllm_omni/host_weight_runtime/ # Direct review destination for every structural owner above. The audit checks # that each route has a real current knowledge entrypoint. owner_documents: @@ -111,6 +113,8 @@ owner_documents: runtime-core: - knowledge/repos/vllm-omni/components/model-executor/rules.md + host-weight-runtime: + - knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md ignored_paths: - pattern: .gitignore reason: repository metadata diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fd3998e75..f39e8e5c7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5730af0a95299387c233e12dfbdbe728aace0e58`. The canonical machine baseline +73b623f2f7db092053c1c86fe796bed89eb3dc71`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1c6f226ff..4018188e0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5730af0a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 73b623f2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1e765f25d..206fd0ed5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5730af0a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 73b623f2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c85738a21..2184a6afe 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5730af0a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 73b623f2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f3b12da35..749556e82 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5730af0a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 73b623f2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6e07e86fd..5046fda61 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5730af0a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 73b623f2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2d76e37da..3d179aa22 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5730af0a` 验证存在 +- 源码校验：以上路径均已在 `main @ 73b623f2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6a3b2407c..a9c6e8b2e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5730af0a`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 73b623f2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `37c11e6e` 本轮新增 Omni Chat Completions 的 prompt token details 合同：启用详情时，streaming 与 non-streaming 都必须保留 `cached_tokens: 0` 并传播多模态 token 计数。该规则不代表所有 endpoint 或 Prometheus 指标都采用相同字段，也不覆盖模型专属 token 统计语义。 +本次相对 `5730af0a` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 37c11e6e \ - --to 5730af0a \ + --from 5730af0a \ + --to 73b623f2 \ --repo \ --mode report-only ``` From cb386bcbd3821614728a43ccb7c621669ee2db57 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:16:43 +0800 Subject: [PATCH 245/465] knowledge(vllm-omni): sync 70bdd494 (PR #5194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 70bdd49402004169b7d93f7f8620f7325d3b0b0e Upstream-PR: #5194 Knowledge-Delta: 新增 HiDream-O1 Image 的模型专属请求/并行/输出合同，并沉淀共享 mixed-attention 与 checkpoint 自动解析门禁。 --- adapters/vllm_omni/release_baseline.yaml | 8 +++---- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-attention.md | 9 +++++++- .../diffusion/rules-system-runtime.md | 9 +++++++- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 +++---- .../models/hidream-o1-image/_index.md | 2 ++ .../models/hidream-o1-image/rules.md | 22 +++++++++++++++++++ 13 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/hidream-o1-image/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 81858efc1..5dd3a3903 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5730af0a95299387c233e12dfbdbe728aace0e58 + previous_audited_sha: 73b623f2f7db092053c1c86fe796bed89eb3dc71 audited_ref: main - audited_sha: 73b623f2f7db092053c1c86fe796bed89eb3dc71 + audited_sha: 70bdd49402004169b7d93f7f8620f7325d3b0b0e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 88 sha256: 96d973728f1d67a959121c4aa0d8ba86cff32043ec6f90458f48d1d766acb8af diffusion: - count: 66 - sha256: cc2a3220f4808990f9a164c8944d3b8b7ace7217599ac975fb18e9719d53029a + count: 67 + sha256: 9c7160d18bdfff9158738a6ea74a280d52d541a45d9c298892773278594fe68f pipelines: count: 59 sha256: c2516b9fb6124aa0774a05390a6f05c08ed6998e287b03c8252d85338934933e diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f39e8e5c7..59fcfc62d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -73b623f2f7db092053c1c86fe796bed89eb3dc71`. The canonical machine baseline +70bdd49402004169b7d93f7f8620f7325d3b0b0e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4018188e0..db1998dc5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 73b623f2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 70bdd494` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 206fd0ed5..590184266 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 73b623f2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 70bdd494` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index f98f3f76a..089cc2880 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614"] +sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py"] confidence: high --- @@ -91,6 +91,13 @@ confidence: high - 禁止：把单 token 形状交给 cuDNN 后等待 eager exception；用包含 FLASH/MATH 的 priority list 掩盖 backend 选择；因 Q/K 长度不同而丢弃已有 attention mask。 - 验收：CPU/mock 参数化覆盖 `(1,1)`、`(8,1)`、`(1,8)` 与 `(8,8)`，精确断言选择的 SDP backend；另覆盖 compile、masked unequal-length 与 unmasked native path，并说明真实 GPU 数值仍需独立验证。^[PR #6070] +## DIFF-1z — 混合 causal/full attention 必须经过共享 Attention 能力路径 + +- 触发：diffusion 模型包含文本 causal 行与 image/timestep full-attention 行的混合序列，或修改 `full_attn_spans`、attention mask 与高分辨率长序列 dispatch。 +- 强制：通过 `vllm_omni.diffusion.attention.layer.Attention` 传递 `AttentionMetadata`；piecewise backend 消费连续的 `full_attn_spans`，不支持时才使用按请求构造并复用的 dense mask，且 mask 形状必须覆盖完整 text+image 序列。 +- 禁止：在模型内直接调用 raw `scaled_dot_product_attention`，绕过 diffusion attention backend/configuration；把所有 full rows 当作任意离散 spans；在每个 denoise step 重建只依赖 `token_types` 的 SxS mask，或因 fallback 方便而关闭 backend 能力检查。 +- 验收：CPU/mock 覆盖 contiguous full-span、piecewise path、dense fallback、batch mask shape 与 piecewise/dense 输出一致性；高分辨率请求另验证实际 backend 选择、显存边界和 attention mask 不被重复构造。^[PR #5194] + ## DIFF-7a — 多控制 transfer attention 必须按控制独立计算并显式承担 Ulysses 复制成本 - 触发：修改 Cosmos3 transformer 的多控制 transfer attention、控制 token packing、控制权重传递或其 Ulysses/sequence-parallel 路径。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index 4e1ea5de3..cc4ab11e3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, "PR #5491"] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] confidence: high --- @@ -170,3 +170,10 @@ confidence: high - 禁止：引入独立 block pool、KV spec、refcount 或 physical lifecycle；把部分 CFG 分配当作成功；把临时容量不足误报为永久失败，或让 never-fit 请求在 FIFO 头部无限阻塞；在 DLO DP wave 中拆开 request 与对应 metadata，或让 scheduler/busy-loop 异常逃逸而不唤醒请求流。 - 验收：覆盖 dense 默认旁路、rank/spec/memory mismatch、profile 顺序、CFG partial rollback、临时压力返回 `None` 后 FIFO 重试、空池也放不下时只终止该请求、native allocation error、preemption metadata 保留，以及 finish/cancel/error/pop/close 全部释放；DLO 多 rank RPC 必须逐 envelope 保持 request 与 metadata 配对。 ^[PR #6094] +## DIFF-4t — HiDream-O1 checkpoint signature 必须正向门禁 + +- 触发：diffusion registry、`resolve_model_class_name()`、`OmniDiffusionConfig` enrichment 或 HF checkpoint 自动识别逻辑需要区分 HiDream-O1 与普通 Qwen3-VL。 +- 强制：仅当 `config.json` 的 `model_type` 为 `qwen3_vl`，且 `model.safetensors.index.json` 的原始 `weight_map` 含 `model.final_layer2.linear.weight` 或 `final_layer2.linear.weight` 时，才返回 `HiDreamO1ImagePipeline`；`resolve_model_class_name()` 与 config enrichment 必须复用同一正向 signature，registry 同时登记 pipeline 与 post-process。 +- 禁止：把任意 `qwen3_vl` 配置归类为 HiDream-O1；给 raw index key 强加 `model.` 前缀；只增加显式 `model_class_name` 映射而让通用 text-to-image/server 自动解析仍落到未注册 architecture；缺失或格式异常的 index 不得误报命中。 +- 验收：覆盖 HiDream signature 命中、普通 Qwen3-VL、缺失/错误 `weight_map`、显式类名与 signature 不一致，以及 resolver/enrichment 两条路径；再断言 registry pipeline 和 module-level post-process 均可解析。^[PR #5194] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2184a6afe..905bbf35c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 73b623f2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 70bdd494` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 749556e82..96b610924 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 73b623f2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 70bdd494` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 5046fda61..cf2e72af8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 73b623f2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 70bdd494` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3d179aa22..d9770f5fc 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 73b623f2` 验证存在 +- 源码校验：以上路径均已在 `main @ 70bdd494` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a9c6e8b2e..a4b960e44 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 73b623f2`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 70bdd494`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `5730af0a` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `73b623f2` 本轮新增 `HiDreamO1ImagePipeline` 及 `models/hidream-o1-image` owner 规则。该移动仅表示已接入 BF16 文生图、TP=2 与 Cache-DiT 验证路径，不应推断已支持图片编辑、任意批量、CFG 并行、量化或其他硬件上的性能 parity。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 5730af0a \ - --to 73b623f2 \ + --from 73b623f2 \ + --to 70bdd494 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hidream-o1-image/_index.md b/knowledge/repos/vllm-omni/models/hidream-o1-image/_index.md index 3d0dd58e2..0b28b5315 100644 --- a/knowledge/repos/vllm-omni/models/hidream-o1-image/_index.md +++ b/knowledge/repos/vllm-omni/models/hidream-o1-image/_index.md @@ -51,3 +51,5 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| | 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | + +| HiDream-O1 的请求限制、分辨率、TP、Cache-DiT、mixed attention 与输出验收 | [模型规则](rules.md) | \ No newline at end of file diff --git a/knowledge/repos/vllm-omni/models/hidream-o1-image/rules.md b/knowledge/repos/vllm-omni/models/hidream-o1-image/rules.md new file mode 100644 index 000000000..0285be6d7 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/hidream-o1-image/rules.md @@ -0,0 +1,22 @@ +--- +title: "HiDream-O1 Image 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/registry.py"] +confidence: high +--- + +# HiDream-O1 Image 规则 + +HiDream-O1 Image 的模型专属约束集中在本页；模型 registry signature、共享 attention backend 及通用 Cache-DiT lifecycle 不在此重复展开。 + +## HIDREAMO1-1a — HiDream-O1 Image pipeline 请求、并行与输出合同 + +- 触发：修改 HiDream-O1 Image 的分辨率、prompt/图像 token 构造、采样限制、TP、`_dit_modules`、Cache-DiT 或 patch 输出后处理。 +- 强制：仅支持 BF16、单 prompt、单输出和 `cfg_parallel_size<=1`；将请求尺寸吸附到 `PREDEFINED_RESOLUTIONS`，以 32 为 patch size；文本行保持 causal、timestep/image 行保持 full attention，并遵守 [DIFF-1z](../../components/diffusion/rules-attention.md)。TP 路径必须使用 vLLM parallel layers；`_dit_modules` 固定指向 `model.model.language_model`，Cache-DiT 使用 `ForwardPattern.Pattern_3` 与 `has_separate_cfg=True`；post-process 必须按 patch 布局还原 RGB 图像。 +- 禁止：把任意请求宽高当作 checkpoint 原生尺寸；接受多 prompt、多输出、显式 latents/timesteps、CFG parallel 或图片编辑输入；把 `model.model.language_model` 的 Cache-DiT target 替换成不存在的顶层 transformer；仅凭进程启动或 shape 正确宣称 TP、缓存和输出语义已验证。 +- 验收：覆盖尺寸吸附、BF16/批量/不支持参数的 fail-fast、混合 attention 与 RGB 尺寸检查；以固定 prompt、seed=42、2048x2048、50 steps、guidance=5、TP=2 在 2x H100 上回归无缓存与 Cache-DiT，断言 nested target、输出质量阈值和资源/延迟证据均绑定该 exact case。^[PR #5194] + +共享混合 attention 合同见 [DIFF-1z](../../components/diffusion/rules-attention.md)，自动解析与 registry 合同见 [DIFF-4t](../../components/diffusion/rules-system-runtime.md)。 From 0e3cb0f41140ac3ee8af16a5a6b87ed6fb91c685 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:18:39 +0800 Subject: [PATCH 246/465] knowledge(vllm-omni): sync 15e38ac1 (PR #6427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 15e38ac1730242e09478a6abd8634a0d816834ae Upstream-PR: #6427 Knowledge-Delta: 新增 HWR-1c，明确 post-load publication 的独立报告、策略边界、lease 清理和仅温热未来启动语义。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/host-weight-runtime/rules.md | 11 +++++++++-- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 22 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5dd3a3903..d6dc734d9 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 73b623f2f7db092053c1c86fe796bed89eb3dc71 + previous_audited_sha: 70bdd49402004169b7d93f7f8620f7325d3b0b0e audited_ref: main - audited_sha: 70bdd49402004169b7d93f7f8620f7325d3b0b0e + audited_sha: 15e38ac1730242e09478a6abd8634a0d816834ae # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 59fcfc62d..a29c50bc4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -70bdd49402004169b7d93f7f8620f7325d3b0b0e`. The canonical machine baseline +15e38ac1730242e09478a6abd8634a0d816834ae`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index db1998dc5..ba819f9f3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 70bdd494` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 15e38ac1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 590184266..7f65e1b49 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 70bdd494` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 15e38ac1` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 905bbf35c..1b6f4c543 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 70bdd494` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 15e38ac1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md index 63e3cf9bf..6a1b54ee7 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md @@ -1,10 +1,10 @@ --- title: "Host Weight Runtime 规则" created: 2026-08-23 -updated: 2026-08-23 +updated: 2026-09-04 type: rule tags: [vllm-omni, components] -sources: ["PR #6419", "PR #6445", vllm_omni/host_weight_runtime/] +sources: ["PR #6419", "PR #6445", vllm_omni/host_weight_runtime/, "PR #6427"] confidence: high --- @@ -41,6 +41,13 @@ confidence: high - 验收：线程 close、fork、open-vs-deny race、orphan deny 和 remote/unknown filesystem 回归分别 证明锁、状态和 typed result；cleanup 不移除活跃 mapping。 ^[PR #6419] +## HWR-1c — post-load publication 只温热未来启动 + +- 触发：canonical model 完成加载后需要通过 `POST_LOAD_ONLY` producer 显式温热缺失的 host artifact。 +- 强制：调用同步 `publish_after_load()`，由 runtime 统一执行 store policy；`allow_post_load_publish` 与 pre-load 的 `allow_local_build` 独立控制；成功取得的 validated lease 必须在返回前关闭，并通过独立 publication report 反馈 `PUBLISHED`、`ALREADY_PRESENT` 或 `JOINED`。 +- 禁止：让 `POST_LOAD_ONLY` producer 进入 pre-load `resolve()`；绕过 runtime 直接调用 store；恢复、rebind 或修改当前启动使用的 canonical model；让 post-load failure 改写已完成的 canonical-fallback resolution。 +- 验收：验证 policy-disabled/runtime-disabled 不运行 producer，pre-load 不调用 post-load producer，成功 publication 关闭 lease 且后续 `resolve()` 命中；验证 publication failure、`JOINED`、unexpected store status 和 observer 回调均产生正确的独立 typed report。 ^[PR #6427] + ## HWR-2a — restore 是 validation-only plan 加一次性 commit - 触发：manifest/schema、producer/restorer identity 或模型 hydration 变化。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 96b610924..9cce6546b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 70bdd494` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 15e38ac1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cf2e72af8..03a4d7d29 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 70bdd494` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 15e38ac1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d9770f5fc..7631d084b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 70bdd494` 验证存在 +- 源码校验：以上路径均已在 `main @ 15e38ac1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a4b960e44..653341940 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 70bdd494`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 15e38ac1`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `73b623f2` 本轮新增 `HiDreamO1ImagePipeline` 及 `models/hidream-o1-image` owner 规则。该移动仅表示已接入 BF16 文生图、TP=2 与 Cache-DiT 验证路径，不应推断已支持图片编辑、任意批量、CFG 并行、量化或其他硬件上的性能 parity。 +本次相对 `70bdd494` 本轮将 Host Weight Runtime 的显式 post-load publication、独立结果报告和 warm-only lease 边界沉淀到组件规则中；不得据此推断当前启动会恢复或修改 canonical model，也不得把文档和测试改动视为组件规则。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 73b623f2 \ - --to 70bdd494 \ + --from 70bdd494 \ + --to 15e38ac1 \ --repo \ --mode report-only ``` From b795e9f887b0f5ce1e51c8210441b3687f1cf7d9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:19:12 +0800 Subject: [PATCH 247/465] knowledge(vllm-omni): sync 01686138 (PR #5641) Upstream-Commit: 016861385f3c61b34ad03d5096a05bf0df5631ad Upstream-PR: #5641 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d6dc734d9..8b0cbe753 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 70bdd49402004169b7d93f7f8620f7325d3b0b0e + previous_audited_sha: 15e38ac1730242e09478a6abd8634a0d816834ae audited_ref: main - audited_sha: 15e38ac1730242e09478a6abd8634a0d816834ae + audited_sha: 016861385f3c61b34ad03d5096a05bf0df5631ad # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a29c50bc4..309b12651 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -15e38ac1730242e09478a6abd8634a0d816834ae`. The canonical machine baseline +016861385f3c61b34ad03d5096a05bf0df5631ad`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ba819f9f3..51717fddb 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 15e38ac1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 01686138` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7f65e1b49..10c0ea84f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 15e38ac1` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 01686138` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1b6f4c543..53e5ca039 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 15e38ac1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 01686138` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9cce6546b..e5bce3fd5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 15e38ac1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 01686138` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 03a4d7d29..83ba8e472 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 15e38ac1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 01686138` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7631d084b..15c2715ad 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 15e38ac1` 验证存在 +- 源码校验：以上路径均已在 `main @ 01686138` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 653341940..c9403b6e9 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 15e38ac1`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 01686138`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `70bdd494` 本轮将 Host Weight Runtime 的显式 post-load publication、独立结果报告和 warm-only lease 边界沉淀到组件规则中；不得据此推断当前启动会恢复或修改 canonical model，也不得把文档和测试改动视为组件规则。 +本次相对 `15e38ac1` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 70bdd494 \ - --to 15e38ac1 \ + --from 15e38ac1 \ + --to 01686138 \ --repo \ --mode report-only ``` From dcfc07f3f530d3db0d70c9f2fd22b32d465b509d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:19:46 +0800 Subject: [PATCH 248/465] knowledge(vllm-omni): sync fc163cd9 (PR #6363) Upstream-Commit: fc163cd90f71746c7df112c67b0f3620d7a90ee5 Upstream-PR: #6363 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8b0cbe753..3ddf54967 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 15e38ac1730242e09478a6abd8634a0d816834ae + previous_audited_sha: 016861385f3c61b34ad03d5096a05bf0df5631ad audited_ref: main - audited_sha: 016861385f3c61b34ad03d5096a05bf0df5631ad + audited_sha: fc163cd90f71746c7df112c67b0f3620d7a90ee5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 309b12651..52a773ce0 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -016861385f3c61b34ad03d5096a05bf0df5631ad`. The canonical machine baseline +fc163cd90f71746c7df112c67b0f3620d7a90ee5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 51717fddb..1148ba151 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 01686138` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fc163cd9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 10c0ea84f..a6d9e2246 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 01686138` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ fc163cd9` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 53e5ca039..bd5145229 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 01686138` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fc163cd9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e5bce3fd5..cbf4199f3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 01686138` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fc163cd9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 83ba8e472..8903652c7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 01686138` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fc163cd9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 15c2715ad..e6cf26ef7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 01686138` 验证存在 +- 源码校验：以上路径均已在 `main @ fc163cd9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c9403b6e9..d5d43c6db 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 01686138`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fc163cd9`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `15e38ac1` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `01686138` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 15e38ac1 \ - --to 01686138 \ + --from 01686138 \ + --to fc163cd9 \ --repo \ --mode report-only ``` From e751876b1a26aaa379c6e7b15644290d35bef9c8 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:20:20 +0800 Subject: [PATCH 249/465] knowledge(vllm-omni): sync ea0a2446 (PR #6446) Upstream-Commit: ea0a24465b6ee2d01c87c9b035fa6bc147151b0f Upstream-PR: #6446 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3ddf54967..02e89fc30 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 016861385f3c61b34ad03d5096a05bf0df5631ad + previous_audited_sha: fc163cd90f71746c7df112c67b0f3620d7a90ee5 audited_ref: main - audited_sha: fc163cd90f71746c7df112c67b0f3620d7a90ee5 + audited_sha: ea0a24465b6ee2d01c87c9b035fa6bc147151b0f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 52a773ce0..fe8c8f240 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -fc163cd90f71746c7df112c67b0f3620d7a90ee5`. The canonical machine baseline +ea0a24465b6ee2d01c87c9b035fa6bc147151b0f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1148ba151..7ea20db68 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fc163cd9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ea0a2446` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a6d9e2246..d864bd6f7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fc163cd9` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ea0a2446` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index bd5145229..ad9427281 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fc163cd9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ea0a2446` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cbf4199f3..80e71ec94 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fc163cd9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ea0a2446` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8903652c7..bbb7d2169 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fc163cd9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ea0a2446` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e6cf26ef7..53b4ae16f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fc163cd9` 验证存在 +- 源码校验：以上路径均已在 `main @ ea0a2446` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d5d43c6db..74ca4382f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fc163cd9`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ea0a2446`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `01686138` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `fc163cd9` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 01686138 \ - --to fc163cd9 \ + --from fc163cd9 \ + --to ea0a2446 \ --repo \ --mode report-only ``` From 16495b3e2e4b2972740a6c55ec9d4f3af322ec24 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:24:33 +0800 Subject: [PATCH 250/465] knowledge(vllm-omni): sync 4f7c08c0 (PR #6406) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 4f7c08c0eb0dd2dd47a1baa70384a46c7536afd5 Upstream-PR: #6406 Knowledge-Delta: 新增 async-chunk runtime snapshot 替换、显式 prompt replacement 的 KV/encoder/watermark 清理，以及 MiniCPM-o Code2Wav 重复终端边界规则。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/distributed/rules.md | 9 ++++++++- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-bridge-batch.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md | 9 ++++++++- 13 files changed, 45 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 02e89fc30..f7b94a637 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fc163cd90f71746c7df112c67b0f3620d7a90ee5 + previous_audited_sha: ea0a24465b6ee2d01c87c9b035fa6bc147151b0f audited_ref: main - audited_sha: ea0a24465b6ee2d01c87c9b035fa6bc147151b0f + audited_sha: 4f7c08c0eb0dd2dd47a1baa70384a46c7536afd5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fe8c8f240..4985e7c18 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ea0a24465b6ee2d01c87c9b035fa6bc147151b0f`. The canonical machine baseline +4f7c08c0eb0dd2dd47a1baa70384a46c7536afd5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7ea20db68..f7f9fed4d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ea0a2446` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4f7c08c0` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d864bd6f7..999c937fe 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ea0a2446` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4f7c08c0` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ad9427281..47e555ea9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ea0a2446` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4f7c08c0` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 60c49810b..4b9540ce2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6033", "PR #6360"] +sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6033", "PR #6360", "PR #6406"] confidence: high --- @@ -63,6 +63,13 @@ confidence: high - 禁止：仅凭 `requests_origin_status` 存在就恢复状态；把 resumable `FINISHED_STOPPED` 当作普通 finished 请求而跳过 connector 回收；分散复制部分 receiver 清理逻辑导致重复释放或残留 active-stream 占用；用无关请求的队列状态扩大清理范围。 - 验收：覆盖 connector 隐藏持有、scheduler `running`/`waiting`/`skipped_waiting` 持有、无 connector 所有权及 stale origin；断言隐藏队列、origin ledger、active streams 和 receiver 状态在重复 finish 后仍为空且 active-window 可被新请求重新占用。^[PR #6360] +## DIST-1f — Code2Wav 快照替换必须显式标记并保持终端就绪 + +- 触发：async-chunk connector 收到 generation runtime payload、终端 chunk 或重复 terminal boundary。 +- 强制：只有 producer 显式设置 `replace_runtime_additional_information=True` 才全量替换；Code2Wav 的空 replacement 必须同时带 `is_segment_finished=True` 以标记就绪；替换时删除旧 `codes.audio`，但保留当前 snapshot 的 `codes.ref` 等 sibling 字段。 +- 禁止：把未标记的 generation/diffusion payload 改成替换语义；把 control-only boundary 写成未就绪的空更新；跨 segment merge/replay 旧 terminal audio。 +- 验收：覆盖标记/未标记、audio+ref、空 replacement 和重复 terminal；断言新 prompt、finished/ready 状态正确，旧 audio 不会进入下一次 Code2Wav。 ^[PR #6406] + ## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 - 触发：上游 MoE factory/oracle 或 `init_model_parallel_group` 增加 all-to-all manager 要求。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 80e71ec94..60ee65a9e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ea0a2446` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4f7c08c0` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md index 2cb42e9e0..34e6b369f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666", "PR #5491", "PR #6186", "PR #5452", "vllm_omni/worker/output/payload_build.py"] +sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666", "PR #5491", "PR #6186", "PR #5452", "vllm_omni/worker/output/payload_build.py", "PR #6406"] confidence: high --- @@ -153,3 +153,10 @@ Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x - 禁止：用同时要求输出维度为 1 和 `num_reqs` 的断言限制 batch；构造只含一个 entry 的 payload；用无条件 zero-token early return 遮蔽同步、DP 或 KV-transfer 分支。 - 验收：覆盖两个及以上 request 的 tensor/list 输出、两类长度 mismatch、zero/negative-token、external-launcher + DP、KV no-forward 与 EC producer，分别断言 request 对齐、错误信息和分支调用顺序。 ^[PR #5452] +## EXEC-1p — runtime snapshot 替换必须由 capability 和 producer marker 双重控制 + +- 触发：shared GPU/NPU runner 消费跨 async-chunk 请求和 step 的 runtime additional information。 +- 强制：模型 capability `replace_runtime_additional_information` 必须同时控制 scheduled-new 与 scheduled-cached 路径；替换 helper 写入 request-owned CPU mirror，只保留 `num_processed_tokens`、`resumable` 等 runner bookkeeping；未 opt-in 模型继续使用增量 merge。 +- 禁止：让共享 runner 默认改用全量替换；把旧 terminal payload 合并回新 snapshot；只修 scheduled-new 而遗漏 cached admission；把 runner bookkeeping 当作模型 payload。 +- 验收：覆盖 new/cached replacement、未标记增量 merge、request mirror 和 sibling request 隔离，并确认 NPU 继承路径使用同一替换实现。 ^[PR #6406] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bbb7d2169..0978f880b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ea0a2446` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4f7c08c0` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index de5e838a9..1c951ab5c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6033", "PR #6360"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6033", "PR #6360", "PR #6406"] --- # Scheduler 规则 @@ -278,3 +278,10 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 验收：覆盖 allowlisted code2wav 与 non-allowlisted talker 对照，分别验证普通 deploy 的一次 request-end 全 payload、streaming deploy 的累计 chunk/终端 chunk、connector requeue 和 abort cleanup。 ^[PR #5842] +## SCHED-6d — 显式 prompt replacement 必须一次性释放旧状态并回到 admission + +- 触发：running 或 computed 的 async prompt 显式携带 `replace_streaming_prompt`，或 ready 的 replacement 进入 scheduler。 +- 强制：替换先释放旧 KV/encoder state 一次，清除 in-flight prefill 所有权和 connector watermark；stale fence 以当前 `num_in_flight_tokens` 赋值，再将请求回到 `WAITING` 并走正常 admission。 +- 禁止：只更新 prompt 而继承旧 cache/watermark；把同一 in-flight frame 在更新和 replacement 路径重复计入 stale；绕过 scheduler admission 直接恢复运行。 +- 验收：覆盖 running replacement、ready async replacement 的重复 reset 和正常重新调度；断言 KV/encoder 释放各一次、watermark 清零、stale counter 恰好可排空且首个新 segment frame 不被丢弃。 ^[PR #6406] + diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 53b4ae16f..54ae13064 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ea0a2446` 验证存在 +- 源码校验：以上路径均已在 `main @ 4f7c08c0` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 74ca4382f..2a901a16b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ea0a2446`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4f7c08c0`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `fc163cd9` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `ea0a2446` 本轮将 PR #6406 的可复用约束分别沉淀到 Distributed 的 async-chunk 快照、Scheduler 的 prompt replacement 清理、Model Executor 的 runtime snapshot 合同，并补充 MiniCPM-o 4.5 native duplex 的模型侧边界。测试改动仅作为验证线索，不表示所有 generation/diffusion 路径都改为替换语义，也不能据此推断自动 context rollover 或音频质量改善。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from fc163cd9 \ - --to ea0a2446 \ + --from ea0a2446 \ + --to 4f7c08c0 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index fd1242c05..3d6a12616 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406"] confidence: high --- @@ -256,6 +256,13 @@ confidence: high - 禁止：只依赖 YAML `max_tokens` 维持 turn 边界；在中间 chunk 永久屏蔽 EOS；把终止采样再次作为普通 codec frame 输出。 - 验收：覆盖 prefill 边界元数据、step 24/25 的 mask/force EOS、simplex 不误触发以及多请求 delta/finished 对齐。 ^[PR #6346] +## MCPMO-4d — async chunk 快照替换必须清理旧音频 + +- 触发：MiniCPM-o 4.5 native duplex 遇到重复 `turn_end` 或新的 Code2Wav async segment。 +- 强制：`tts2code2wav_async_chunk` 对重复终端边界发送 ready empty replacement；Code2Wav payload 声明替换 marker；当前 `codes.ref` 等 sibling 可继续使用，上一段 `codes.audio` 必须丢弃。 +- 禁止：重复边界返回未就绪的 control-only 更新；让旧 terminal audio 与新 snapshot merge/replay；把该模型的替换语义外推给未 opt-in 模型。 +- 验收：覆盖 terminal→empty duplicate、audio+ref、scheduled-new/cached runner、prompt replacement 和 `OmniNPUModelRunner` 继承路径；断言旧 audio 消失且边界被标记为 ready。 ^[PR #6406] + ## MCPMO-5a — accuracy 与 duplex perf 证据必须绑定协议、硬件和实际 gate - 触发：修改 MiniCPM-o 4.5 Daily-Omni、Seed-TTS、realtime duplex 数据集/客户端、benchmark From bce6caeedea10c56834fbfbdcf5d7639d08bdef1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:26:38 +0800 Subject: [PATCH 251/465] knowledge(vllm-omni): sync 260db8ba (PR #6072) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 260db8bab5443f52b38b8eb813899a076c18d54c Upstream-PR: #6072 Knowledge-Delta: 新增共享模型级 CPU offload protocol/direct-call cleanup 合同，并记录 MiniMax H3 视觉与音频 VAE 的请求级 residency 合并要求。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-component-lifecycle.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../vllm-omni/models/minimax-h3/rules-deployment.md | 9 ++++++++- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f7b94a637..1d4688e88 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ea0a24465b6ee2d01c87c9b035fa6bc147151b0f + previous_audited_sha: 4f7c08c0eb0dd2dd47a1baa70384a46c7536afd5 audited_ref: main - audited_sha: 4f7c08c0eb0dd2dd47a1baa70384a46c7536afd5 + audited_sha: 260db8bab5443f52b38b8eb813899a076c18d54c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4985e7c18..919895c3b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4f7c08c0eb0dd2dd47a1baa70384a46c7536afd5`. The canonical machine baseline +260db8bab5443f52b38b8eb813899a076c18d54c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f7f9fed4d..10ec5684b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4f7c08c0` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 260db8ba` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 999c937fe..9f357678a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4f7c08c0` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 260db8ba` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md index f4dd71959..f64dff8fe 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5720", "PR #5853", "PR #5884", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py"] +sources: ["PR #5720", "PR #5853", "PR #5884", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py", "PR #6072"] confidence: high --- @@ -87,3 +87,10 @@ residency 留在模型 owner。规则入口与其他共享机制仍见 [Diffusio - 禁止：从 one-stage/multi-stage 名称推断 Cache-DiT 支持；沿用 separate-CFG step counter；把 LTX-2.5 distilled/full 或两阶段 schedule 当成已 qualified 的 Cache-DiT 路径。 - 验收：参数化检查每个 LTX recipe 与 adapter flag，覆盖启用、禁用和 component-init 前 rejection，并验证 fused guidance 每个 denoise step 只推进一次；无真实硬件 Cache-DiT accuracy 证据时不得扩大支持范围。^[PR #6070] +### DIFF-2w — 模型级 CPU offload 的 direct component lifecycle 必须由显式 protocol 管理 + +- 触发：共享 diffusion pipeline 需要在 `forward` 之外直接调用 VAE 或其他 stage，或修改 model-level CPU offload 的启停与 component discovery。 +- 强制：通过显式 `SupportsModelCpuOffload` protocol 委托完整生命周期；custom pipeline 使用 `ModuleDiscovery` 取得实际 DiT、encoder 和 VAE 集合，安装统一 sequential hooks，并可用 `offload_initial_dits=True` 先将 DiT 放回 CPU。所有非 `forward` 的直接 component call 必须由 `sequential_offload_component` 调用 hook 的 `pre_forward`，成功或异常后都执行 `_to_cpu`；禁用时移除全部已注册 module 的 hooks。 +- 禁止：以可调用属性探测替代 protocol、只依赖 generic forward hook 管理 direct VAE call、在没有 activation/release scope 时直接调用 hooked component，或在启停失败后遗留部分 hooks 与设备状态。 +- 验收：CPU/mock 覆盖 protocol pipeline 与普通 pipeline、真实 discovery 的 DiT/stage 列表、初始 DiT CPU residency、direct component activation failure 的 finally cleanup，以及 enable/disable 后所有 hook 清理；真实 accelerator 的性能与完整模型显存结论仍需独立验证。^[PR #6072] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 47e555ea9..ebff499a0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4f7c08c0` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 260db8ba` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 60ee65a9e..47edfc992 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4f7c08c0` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 260db8ba` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0978f880b..e7ecf2fc1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4f7c08c0` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 260db8ba` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 54ae13064..7818e4d6f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4f7c08c0` 验证存在 +- 源码校验：以上路径均已在 `main @ 260db8ba` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2a901a16b..02c2729b6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4f7c08c0`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 260db8ba`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `ea0a2446` 本轮将 PR #6406 的可复用约束分别沉淀到 Distributed 的 async-chunk 快照、Scheduler 的 prompt replacement 清理、Model Executor 的 runtime snapshot 合同，并补充 MiniCPM-o 4.5 native duplex 的模型侧边界。测试改动仅作为验证线索，不表示所有 generation/diffusion 路径都改为替换语义，也不能据此推断自动 context rollover 或音频质量改善。 +本次相对 `4f7c08c0` 本轮为共享 diffusion offloader 增加模型级 CPU offload protocol 与非 forward component 的异常安全生命周期规则，并为 MiniMax H3 增加视觉/音频 VAE residency window 规则。未新增页面或索引行。不得据此推断完整模型峰值显存、通用性能收益或所有模型的 offload 兼容性。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from ea0a2446 \ - --to 4f7c08c0 \ + --from 4f7c08c0 \ + --to 260db8ba \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index 8dad7f21c..f22e48eb0 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py"] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py", "PR #6072"] confidence: high --- @@ -229,3 +229,10 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule - 禁止：让无 tile 的 rank 抛错退出 collective、让其他 rank 留在 `all_gather`；不得使用 encoder grid 代替 decoder grid、让各 rank 依据不同条件分支，或把 rank-local fallback 描述成无性能代价或通用 VAE 并行支持。 - 验收：CPU 回归必须实际调用 `decode_latent`，覆盖 decoder tile count 1、2、4、28、112 及 2/4、4/4、2/2、`parallel_size=1` 边界，断言 decode 观察到的 rank-local state，并验证正常与异常退出恢复完整 group state；4-rank 硬件复现中 2 tiles 必须正常退出，4 tiles 与 shipped `48x84` latent（28 tiles）必须保持并行路径。^[PR #6345] +## MMH3-3l — H3 直接 VAE 调用必须共享请求级 residency window + +- 触发：MiniMax H3 在 model-level CPU offload 下处理 image/video reference 或 embedded/standalone audio reference，且 pipeline 直接调用 Video-VAE 或 Audio-VAE。 +- 强制：`_encode_visual_conditions` 必须在同一个 `video_vae` residency window 内完成该请求的全部 image 与 video encode；`_encode_reference_audio_conditions` 必须在同一个 `audio_vae` residency window 内完成 embedded 与 standalone audio encode。resident helper 只能在对应 scope 内调用，scope 退出时必须释放 component，异常也必须清理。 +- 禁止：按 image、video 或 audio 条目分别建立重复的 H2D/D2H lifecycle；在前一个 scope 结束后直接调用 CPU-bound VAE；把单次 image 的行为外推为混合 reference，或把 transfer-level A/B 结果写成完整模型峰值显存与通用性能保证。 +- 验收：contract tests 覆盖单/多 image、image+video、embedded+standalone audio 以及 encode exception，精确断言每个 VAE 只有一次 activate/offload；另验证 H3 model-level offload 注册所有实际 DiT 与 VAE stage，并以固定硬件和 workload 单独复核真实性能与输出质量。^[PR #6072] + From 742505600313abd46fa4fbc4d6b065b1ac067a09 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:29:12 +0800 Subject: [PATCH 252/465] knowledge(vllm-omni): sync dc72559d (PR #6454) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: dc72559d5d975e4ba2f60461455d6b3cfeac7102 Upstream-PR: #6454 Knowledge-Delta: 补充 CosyVoice3 causal HiFT 的运行时 STFT window 必须注册为非持久 buffer，并随 module 的 device/dtype 迁移的验收合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-runtime-hot-paths.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1d4688e88..d47d9d49c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4f7c08c0eb0dd2dd47a1baa70384a46c7536afd5 + previous_audited_sha: 260db8bab5443f52b38b8eb813899a076c18d54c audited_ref: main - audited_sha: 260db8bab5443f52b38b8eb813899a076c18d54c + audited_sha: dc72559d5d975e4ba2f60461455d6b3cfeac7102 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 919895c3b..397bf9143 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -260db8bab5443f52b38b8eb813899a076c18d54c`. The canonical machine baseline +dc72559d5d975e4ba2f60461455d6b3cfeac7102`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 10ec5684b..a7b109e3a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 260db8ba` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ dc72559d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9f357678a..2c13614e4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 260db8ba` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ dc72559d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ebff499a0..7e4eae91f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 260db8ba` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ dc72559d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 47edfc992..f32150c61 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 260db8ba` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ dc72559d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md index 6be935374..052d7e9a3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048"] +sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048", "PR #6454", "vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py"] confidence: high --- @@ -63,3 +63,10 @@ confidence: high - 禁止：把 mask 应用于整个 batch、假定普通采样总会填充 history、用本地 counter 在恢复后重启 ban，或在 spec decode 的多行映射下按位置读取 mode；不得静默吞掉形状不一致而继续按错误 request masking。 - 验收：覆盖默认关闭、step `0..N-1`、`step >= N`、mixed-step/mixed-mode batch、ICL/未记录 mode、缺失或空 `output_token_ids`、history/req_id mismatch、warning 去重与 preemption history 保留；确认无早期目标时不产生热路径 device allocation。^[PR #5048] +## EXEC-11f — 音频 STFT 运行时窗口必须随模块迁移 + +- 触发：模型的声码器或 HiFT 等 STFT 路径在 module `.to(...)`、多设备 stage 或 CUDA 推理中使用由 `__init__` 创建的运行时 tensor，尤其是子类绕过基类初始化时。 +- 强制：将不属于 checkpoint 的运行时 tensor 通过 `register_buffer(..., persistent=False)` 注册到实际消费它的 module，使 device/dtype 迁移覆盖所有初始化路径，并保持其与 STFT 输入同设备。 +- 禁止：把设备敏感 tensor 留作普通 attribute，依赖每次调用临时 `.to(device)` 兜底，或只修基类初始化而遗漏 causal/subclass 路径；不得把派生 window 写入 checkpoint state dict。 +- 验收：构造 CosyVoice3 `CausalHiFTGenerator`，执行 `.to(cuda)` 后断言 `stft_window.device` 与 STFT 输入一致并完成 causal inference；CPU 路径仍可运行，`persistent=False` buffer 不出现在 checkpoint state dict，并覆盖 subclass constructor。 ^[PR #6454] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e7ecf2fc1..cda607391 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 260db8ba` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ dc72559d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7818e4d6f..52e35d99f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 260db8ba` 验证存在 +- 源码校验：以上路径均已在 `main @ dc72559d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 02c2729b6..ed85bcde7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 260db8ba`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ dc72559d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `4f7c08c0` 本轮为共享 diffusion offloader 增加模型级 CPU offload protocol 与非 forward component 的异常安全生命周期规则，并为 MiniMax H3 增加视觉/音频 VAE residency window 规则。未新增页面或索引行。不得据此推断完整模型峰值显存、通用性能收益或所有模型的 offload 兼容性。 +本次相对 `260db8ba` 本轮补充 CosyVoice3 causal HiFT 的 STFT window 设备迁移合同，归档至 model-executor 运行时热路径规则。该规则不代表所有模型子模块或平台都具有相同的设备与精度合同。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 4f7c08c0 \ - --to 260db8ba \ + --from 260db8ba \ + --to dc72559d \ --repo \ --mode report-only ``` From b1c0d829e4f8ea6d9e3f6e9382492ed9dc993edb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:31:20 +0800 Subject: [PATCH 253/465] knowledge(vllm-omni): sync cfbf1392 (PR #6467) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: cfbf1392c5b97c81d0189a60f54b2798e7935401 Upstream-PR: #6467 Knowledge-Delta: 新增 Step-Audio2 音频加载、mel filter 和独立 16/24 kHz 重采样必须走共享 vLLM audio runtime 的审查合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/step-audio2/rules.md | 11 +++++++++-- 10 files changed, 22 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d47d9d49c..52c26041c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 260db8bab5443f52b38b8eb813899a076c18d54c + previous_audited_sha: dc72559d5d975e4ba2f60461455d6b3cfeac7102 audited_ref: main - audited_sha: dc72559d5d975e4ba2f60461455d6b3cfeac7102 + audited_sha: cfbf1392c5b97c81d0189a60f54b2798e7935401 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 397bf9143..c869feaea 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -dc72559d5d975e4ba2f60461455d6b3cfeac7102`. The canonical machine baseline +cfbf1392c5b97c81d0189a60f54b2798e7935401`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a7b109e3a..b9ae50519 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ dc72559d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ cfbf1392` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2c13614e4..f6d1a2271 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ dc72559d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ cfbf1392` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7e4eae91f..508951ae5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ dc72559d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ cfbf1392` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f32150c61..acbb7cb4f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ dc72559d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ cfbf1392` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cda607391..d4cd83a3c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ dc72559d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ cfbf1392` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 52e35d99f..40ebf8bf9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ dc72559d` 验证存在 +- 源码校验：以上路径均已在 `main @ cfbf1392` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ed85bcde7..354771b34 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ dc72559d`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ cfbf1392`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `260db8ba` 本轮补充 CosyVoice3 causal HiFT 的 STFT window 设备迁移合同，归档至 model-executor 运行时热路径规则。该规则不代表所有模型子模块或平台都具有相同的设备与精度合同。 +本次相对 `dc72559d` 本轮在 Step-Audio2 规则页新增音频加载、mel filter 与双采样率重采样的共享 runtime 合同。不可据此推断 onnxruntime、s3tokenizer 等其他依赖已移除，或所有平台均已验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 260db8ba \ - --to dc72559d \ + --from dc72559d \ + --to cfbf1392 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/step-audio2/rules.md b/knowledge/repos/vllm-omni/models/step-audio2/rules.md index 320b801d2..a85ff2a09 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/rules.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/rules.md @@ -1,10 +1,10 @@ --- title: "Step-Audio2 设备边界与流式性能规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models] -sources: ["PR #5067", vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, tests/model_executor/models/step_audio2/test_step_audio2_token2wav_async_chunk.py, tests/platforms/npu/test_step_audio2_token2wav.py] +sources: ["PR #5067", vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, tests/model_executor/models/step_audio2/test_step_audio2_token2wav_async_chunk.py, tests/platforms/npu/test_step_audio2_token2wav.py, "PR #6467"] --- # Step-Audio2 设备边界与流式性能规则 @@ -55,3 +55,10 @@ sources: ["PR #5067", vllm_omni/model_executor/models/step_audio2/step_audio2_th MiniCPM-o 4.5/NPU wrapper 使用。MiniCPM 显式传 list，继续走旧分支； Step-Audio2 NPU wrapper 转发 tensor 并继承新分支。评审共享 core 时 不得把两者当作同样的性能受益方。 + +## STEPA2-2a — ：音频输入与重采样必须复用共享 runtime + +- 触发：修改 Step-Audio2 thinker/token2wav 的音频读取、mel filter 或 16/24 kHz 重采样路径，或调整其音频依赖。 +- 强制：音频读取统一调用 `load_audio(..., sr=None, mono=True)`，mel filter 统一调用 `mel_filter_bank`；16 kHz 的 s3tokenizer/说话人分支和 24 kHz flow-mel 分支都必须从原始 waveform 分别经 `AudioResampler` 生成，保留真实 sample rate 与单声道语义。 +- 禁止：在目标运行时重新导入 `librosa` 或 `soundfile`，通过 try/except 在两套解码实现间兜底，或把已经 16 kHz 的结果再次作为 24 kHz 重采样输入；不得据此声称 onnxruntime/s3tokenizer 等其他依赖已移除。 +- 验收：静态检查 Step-Audio2 runtime 不再依赖 `librosa`/`soundfile`；覆盖非 16/24 kHz 输入，断言两个目标采样率都由同一原始音频独立生成、mel shape/数值和 token2wav 推理结果正常，并运行 `pytest tests/e2e/online_serving/test_step_audio2_expansion.py --run-level 'advanced_model'`。 ^[PR #6467] From 8da57b162116a51b7bbf60eee246b76426dfae6d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:36:52 +0800 Subject: [PATCH 254/465] knowledge(vllm-omni): sync 3d34b8b9 (PR #6181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 3d34b8b9771e068440a91e0367e4aab9e917c2fe Upstream-PR: #6181 Knowledge-Delta: 新增 diffusion DIFF-8a：TeaCache extractor 与 CFG 正负分支必须保持 forward 参数合同一致并显式传递 guidance。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules-attention.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 52c26041c..b4aaf2761 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: dc72559d5d975e4ba2f60461455d6b3cfeac7102 + previous_audited_sha: cfbf1392c5b97c81d0189a60f54b2798e7935401 audited_ref: main - audited_sha: cfbf1392c5b97c81d0189a60f54b2798e7935401 + audited_sha: 3d34b8b9771e068440a91e0367e4aab9e917c2fe # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c869feaea..9a5e9c6a2 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -cfbf1392c5b97c81d0189a60f54b2798e7935401`. The canonical machine baseline +3d34b8b9771e068440a91e0367e4aab9e917c2fe`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b9ae50519..8ed754d11 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ cfbf1392` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3d34b8b9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f6d1a2271..7464f0b97 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ cfbf1392` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 3d34b8b9` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 089cc2880..e1b8ef3f2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py"] +sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] confidence: high --- @@ -104,3 +104,10 @@ confidence: high - 强制：多控制 attention 必须为每个 control 独立构造 `[text, control_i, target]` 的 K/V 序列；保留该 control 自己的输出，target 输出按已归一化的 control weights 加权后再拼回控制区与 target 区。control token size 与 weight 数量必须一一对应，所有 control range 必须为正且完整覆盖 packed control 区。该路径使用 `skip_sequence_parallel=True` 的专用 attention，multi-control 时不得执行 `gen_sp_prepare`/`gen_sp_gather`，以完整的 `[control_i | target]` 序列在每个 Ulysses rank 复制执行，并显式告警其不降低每-rank memory 或 latency；未提供权重时使用均匀相对权重。 - 禁止：把多个 control 直接拼成一个共享 attention 序列后让一个 control 污染其他 control 的输出；按 control token 数对拼接序列做 Ulysses 切分；只验证 output shape 就宣称 sequence parallel 为多控制 transfer 提供 per-rank 节省，或把一个控制的局部测试外推为多控制数值/性能 parity。 - 验收：CPU/mock 测试用至少两个 control 的不同 value 和非均匀权重断言 target 是加权结果、control 输出来自各自 pass，并覆盖 token-size/weight 长度及非法权重；带 Ulysses 的测试断言专用 attention、复制执行和 warning，真实多卡再与单卡 dense reference 对照并分别测量显存、延迟和输出质量。^[PR #5614] + +## DIFF-8a — CFG 双分支与 TeaCache extractor 必须保持参数合同一致 + +- 触发：修改 diffusion transformer 的 `forward` 参数、TeaCache `extract_*_context` hook、CFG 正/负分支 kwargs 或 `guidance_scale > 1` 的执行路径。 +- 强制：以被 hook 的实际 `forward` 签名作为参数合同；extractor 必须兼容真实调用的可选字段，CFG 正负分支必须传递全部 required conditioning kwargs，尤其是 `guidance`，并保持对应 tensor 的语义与形状一致。 +- 禁止：只让正分支携带 extractor 所需参数；假设 negative branch 可以省略 `guidance`；用宽泛 `**kwargs` 或不启用 CFG 的测试掩盖 hook 与 model `forward` 的签名漂移。 +- 验收：TeaCache 启用且 `guidance_scale > 1` 时分别覆盖 LongCat T2I 与 Edit 的正/负分支，断言 extractor 两次均收到 `guidance` 且不抛 `TypeError`；再以固定输入、seed 和 checkpoint 对照无缓存输出。^[PR #6181] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 508951ae5..2b1f859a1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ cfbf1392` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3d34b8b9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index acbb7cb4f..095e400e7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ cfbf1392` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3d34b8b9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d4cd83a3c..2013ca68d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ cfbf1392` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3d34b8b9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 40ebf8bf9..0c444af61 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ cfbf1392` 验证存在 +- 源码校验：以上路径均已在 `main @ 3d34b8b9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 354771b34..67bd727af 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ cfbf1392`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3d34b8b9`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `dc72559d` 本轮在 Step-Audio2 规则页新增音频加载、mel filter 与双采样率重采样的共享 runtime 合同。不可据此推断 onnxruntime、s3tokenizer 等其他依赖已移除，或所有平台均已验证。 +本次相对 `cfbf1392` 本轮将 PR #6181 的 LongCat TeaCache CFG 双分支 guidance 参数契约沉淀到 diffusion attention/cache 规则中。测试框架扩展和 LongCat 专属 text_processor 加载细节未从中推断为共享规则。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from dc72559d \ - --to cfbf1392 \ + --from cfbf1392 \ + --to 3d34b8b9 \ --repo \ --mode report-only ``` From f4731d73e62e11c223427ba63e3ef6240efca13e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:40:06 +0800 Subject: [PATCH 255/465] knowledge(vllm-omni): sync 4882a36c (PR #6288) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 4882a36c41c481f70a3211c72f4e36e7bc800983 Upstream-PR: #6288 Knowledge-Delta: 新增非流式视频响应的自动 planar/legacy 能力门禁，以及共享 PyAV mux 的资源清理和音频时间零点合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-output-lifecycle.md | 8 +++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 28 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b4aaf2761..0dea2af72 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: cfbf1392c5b97c81d0189a60f54b2798e7935401 + previous_audited_sha: 3d34b8b9771e068440a91e0367e4aab9e917c2fe audited_ref: main - audited_sha: 3d34b8b9771e068440a91e0367e4aab9e917c2fe + audited_sha: 4882a36c41c481f70a3211c72f4e36e7bc800983 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9a5e9c6a2..be26492ec 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3d34b8b9771e068440a91e0367e4aab9e917c2fe`. The canonical machine baseline +4882a36c41c481f70a3211c72f4e36e7bc800983`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8ed754d11..3d7fd8513 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3d34b8b9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4882a36c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7464f0b97..d12359c4b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3d34b8b9` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4882a36c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 8ad6455a4..0f97b54cb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094"] +sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6288"] confidence: high --- @@ -109,3 +109,9 @@ confidence: high - 禁止：沿用 image 默认类型、用输出 ndarray 形状猜测语义、把 action 路由成 image/video，或把局部 closure 作为跨进程 post-process hook。 - 验收：registry 能解析 pipeline 与 post-process callable，callable 可被目标进程边界使用，最终 engine output 保留稳定 key，并由端到端测试断言 action shape、dtype 和有限性。^[PR #4222] +## DIFF-9a — 共享 PyAV 预构造帧 mux 必须闭合资源与音频时间零点 + +- 触发：修改共享 diffusion 的预构造 PyAV 视频帧 mux、可选音频 mux、container cleanup 或编码器异常传播。 +- 强制：`mux_av_video_audio_bytes()` 必须在 context manager 内打开并配置 MP4 video/audio streams，消费 `Iterable[av.VideoFrame]` 后 flush video encoder；音频按 `fltp` 和 mono/stereo layout 构造，并以 `pts=0`、`time_base=1/sample_rate` 标记输入时间零点。成功和 generator、encoder、mux 异常都必须退出 context 关闭 container，同时原样传播原始错误。 +- 禁止：在 generator 消费前或 context 外打开/使用 container；只在成功路径手工 close；吞掉帧生成或编码异常后静默改走其他 muxer；把 mock container cleanup 或单一 CPU 运行误认为完整跨硬件 A/V parity。 +- 验收：测试覆盖预构造 `gbrp` 帧的 H.264 mux、无音频/mono/stereo 音频、AAC 首个有效 packet 的负 priming PTS、video flush 和 container close；让 frame generator 在中途抛错并断言 container 已关闭且异常 identity 保持，再用固定媒体输入执行 ffprobe 与完整 FFmpeg 解码。^[PR #6288] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2b1f859a1..f48132e74 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3d34b8b9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4882a36c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 095e400e7..991bcaf5b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3d34b8b9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4882a36c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2013ca68d..29d6f9c2f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3d34b8b9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4882a36c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0c444af61..88884b7c9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3d34b8b9` 验证存在 +- 源码校验：以上路径均已在 `main @ 4882a36c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index af39214f9..5c200a42d 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288"] confidence: high --- @@ -147,6 +147,13 @@ confidence: high - 禁止：把 usage JSON 塞入 raw-audio body；用原始 stage prefill 长度替代共享 usage 语义；在 streaming headers 已发送后补发尚未确定的最终 totals；宣称 diffusion-mode 路径必然提供这些 headers。 - 验收：测试非流式请求返回 `200`、raw audio body 未改变且五个 headers 的值与 `SpeechTokenUsage` 精确一致；覆盖 usage 不可用时不生成 headers，并回归 streaming 与 batch 的 usage 语义及 diffusion-mode 的无 headers 行为。^[PR #4499] +## SERV-1k — 非流式 MP4 编码必须按运行时能力自动选择并保留回退语义 + +- 触发：修改非流式 MP4 视频响应的 normalize、frame layout/dtype 判断、mux、raw/base64 输出或 fallback 行为。 +- 强制：公共 dispatcher 先完成一次帧准备和形状校验；仅当所有帧具有相同的正 HWC 形状、3/4 通道、支持的 common dtype（`uint8`、bool 或浮点）且 RGB channel view 均为 C-contiguous 时选择 `direct_planar`，逐通道量化到 PyAV 的 GBR planar 帧并复用单个 scratch buffer；其他输入必须在打开 PyAV container 前选择 `legacy_fallback`，复用已准备帧并记录稳定 fallback reason 与有效媒体参数。raw MP4 和 base64 共用该 dispatcher，不提供 CLI、模型声明或请求级编码策略；流式 fMP4 保持原路径。 +- 禁止：为能力探测再次 normalize/copy 视频；在 direct mux 开始后因异常重试 legacy；把 interleaved、非法形状或不支持 dtype 静默送入 planar path；把 Kunpeng/Atlas 测量外推为通用硬件性能或 DiT 加速。 +- 验收：覆盖 channel-first-backed direct path、interleaved/unsupported shape/dtype fallback、RGB contiguity、GBR plane 与 padded stride、direct failure 不重试、音频存在且采样率缺省时仅一次使用 24 kHz 日志、显式采样率和 video-only 日志；固定输入下有无音频的 direct/legacy MP4 必须 byte-identical 并通过 ffprobe/完整 FFmpeg decode，raw/base64 不携带 policy config，streaming 行为不变。^[PR #6288] + ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 67bd727af..9101914ff 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3d34b8b9`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4882a36c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `cfbf1392` 本轮将 PR #6181 的 LongCat TeaCache CFG 双分支 guidance 参数契约沉淀到 diffusion attention/cache 规则中。测试框架扩展和 LongCat 专属 text_processor 加载细节未从中推断为共享规则。 +本次相对 `3d34b8b9` 本轮新增视频响应编码的运行时能力选择与共享 PyAV mux 生命周期规则。不可据此推断特定 CPU 之外的固定加速比、GPU/DiT/stage-0 加速或流式 fMP4 行为改变；CLI、模型声明和请求参数均无需新增。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from cfbf1392 \ - --to 3d34b8b9 \ + --from 3d34b8b9 \ + --to 4882a36c \ --repo \ --mode report-only ``` From fe29a99fa93ea7aafdab32ac59f75f646bc759ed Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:40:38 +0800 Subject: [PATCH 256/465] knowledge(vllm-omni): sync faa12af4 (PR #6459) Upstream-Commit: faa12af454335fc5627f8143760b7ab70e41e875 Upstream-PR: #6459 Knowledge-Delta: baseline-and-pins-only --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0dea2af72..7801af908 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3d34b8b9771e068440a91e0367e4aab9e917c2fe + previous_audited_sha: 4882a36c41c481f70a3211c72f4e36e7bc800983 audited_ref: main - audited_sha: 4882a36c41c481f70a3211c72f4e36e7bc800983 + audited_sha: faa12af454335fc5627f8143760b7ab70e41e875 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index be26492ec..5721966db 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4882a36c41c481f70a3211c72f4e36e7bc800983`. The canonical machine baseline +faa12af454335fc5627f8143760b7ab70e41e875`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3d7fd8513..8ff33fba0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4882a36c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ faa12af4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d12359c4b..9ac783532 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4882a36c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ faa12af4` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f48132e74..94b048a5b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4882a36c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ faa12af4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 991bcaf5b..0341bc307 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4882a36c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ faa12af4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 29d6f9c2f..fd3389ab3 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4882a36c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ faa12af4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 88884b7c9..7ca2afe5a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4882a36c` 验证存在 +- 源码校验：以上路径均已在 `main @ faa12af4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9101914ff..4d1e54572 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4882a36c`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ faa12af4`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `3d34b8b9` 本轮新增视频响应编码的运行时能力选择与共享 PyAV mux 生命周期规则。不可据此推断特定 CPU 之外的固定加速比、GPU/DiT/stage-0 加速或流式 fMP4 行为改变；CLI、模型声明和请求参数均无需新增。 +本次相对 `4882a36c` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 3d34b8b9 \ - --to 4882a36c \ + --from 4882a36c \ + --to faa12af4 \ --repo \ --mode report-only ``` From 9d6de91a85b3c7b0a14b823c7b5432be9e064e23 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:43:26 +0800 Subject: [PATCH 257/465] knowledge(vllm-omni): sync 42059f19 (PR #6458) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 42059f19d5dc981eec7ba95085bd5380beb85d66 Upstream-PR: #6458 Knowledge-Delta: 新增 MiniCPM-o 4.5 Talker 的 16-frame codec penalty、forced EOS 恢复与离线 2048/context cap 规则，并冻结四份 deploy profile 的 Stage 1 采样默认值。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-runtime-hot-paths.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md | 2 +- 12 files changed, 30 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7801af908..d2de4cada 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4882a36c41c481f70a3211c72f4e36e7bc800983 + previous_audited_sha: faa12af454335fc5627f8143760b7ab70e41e875 audited_ref: main - audited_sha: faa12af454335fc5627f8143760b7ab70e41e875 + audited_sha: 42059f19d5dc981eec7ba95085bd5380beb85d66 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5721966db..0f9383ea9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -faa12af454335fc5627f8143760b7ab70e41e875`. The canonical machine baseline +42059f19d5dc981eec7ba95085bd5380beb85d66`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8ff33fba0..65b48359a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ faa12af4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 42059f19` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 95f75341c..973f4633b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458"] --- # vLLM-Omni 配置开发门禁 @@ -172,6 +172,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：只注册 pipeline 或放置 YAML 就依赖 `model_type` 推断；用 `hf_overrides.model_type` 替代 stage architecture 路由；要求离线入口额外传参来弥补默认配置未接通。 - 验收：真实无显式 deploy 配置的初始化能读回最终 pipeline 与 architecture，backbone `get_text_config()` 仍解析为原模型类型，并覆盖 registry、默认 YAML 与最终 stage config 的一致性。 ^[PR #5666] +### CONF-3e — MiniCPM-o 4.5 Stage 1 默认采样参数必须与模型语义一致 + +- 触发：修改 `minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml`、`minicpmo_4_5_3gpu.yaml` 或 `minicpmo_4_5_8x4090.yaml` 的 Stage 1 Talker 默认采样参数。 +- 强制：四份 profile 的 Stage 1 默认值保持一致：`temperature=0.8`、`top_p=0.85`、`top_k=25`、`repetition_penalty=1.05`、`min_tokens=50`、`max_tokens=4096`；duplex overlay 可按 `generate_chunk` 合同将 `min_tokens` 设为 `0`，而 Talker 离线逻辑仍独立限制为 2048 或剩余上下文。 +- 禁止：只更新一个 profile；把旧的 whole-stream penalty 值 `1.02` 沿用到 16-frame frequency penalty；将 Sampler ceiling `4096` 解释为离线 Talker 的实际生成预算；把 `min_p`、`win_size` 或 `tau_r` 当作已由上游 `gen_logits()` 消费的必需字段。 +- 验收：展开四份 MiniCPM-o 4.5 deploy 配置，逐一断言 Stage 1 的采样字段与上游默认一致，并单独验证 duplex overlay 的 `min_tokens=0` 不改变其余继承值；真实 Talker 测试还需确认最终配置到达 Sampler 与模型预算逻辑。^[PR #6458] + ### CONF-6a — MOSS-TTS-Local 部署必须采用官方采样参数 - 触发：修改 MOSS-TTS-Local 的 deploy YAML 或其默认采样参数时。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9ac783532..633febbe6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ faa12af4` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 42059f19` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 94b048a5b..1c94d03f6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ faa12af4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 42059f19` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0341bc307..8d6a72acc 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ faa12af4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 42059f19` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md index 052d7e9a3..07e41005e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048", "PR #6454", "vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py"] +sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048", "PR #6454", "vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py", "PR #6458"] confidence: high --- @@ -70,3 +70,10 @@ confidence: high - 禁止：把设备敏感 tensor 留作普通 attribute，依赖每次调用临时 `.to(device)` 兜底，或只修基类初始化而遗漏 causal/subclass 路径；不得把派生 window 写入 checkpoint state dict。 - 验收：构造 CosyVoice3 `CausalHiFTGenerator`，执行 `.to(cuda)` 后断言 `stft_window.device` 与 STFT 输入一致并完成 causal inference；CPU 路径仍可运行，`persistent=False` buffer 不出现在 checkpoint state dict，并覆盖 subclass constructor。 ^[PR #6454] +## EXEC-11g — MiniCPM-o Talker codec penalty 与终止预算必须在 Sampler 前后闭环 + +- 触发：修改 MiniCPM-o 4.5 Talker 的 codec 采样、repetition penalty、EOS/min_tokens 处理或离线生成长度预算。 +- 强制：在 Stage 1 Sampler 前按请求维护最近 16 个 codec frame，使用请求级 `repetition_penalty` 执行 frequency-aware penalty，并将 Sampler 的同一 penalty 置为 `1` 避免重复计分；离线生成上限取 `min(2048, remaining Talker context)`。对 `compute_logits` 强制的 codec EOS 必须在 `MinTokensLogitsProcessor` 之后恢复到对应采样行；native duplex 仍按既有 `generate_chunk` 的 26-sample 预算执行。 +- 禁止：把 vLLM whole-stream presence penalty 当作上游 16-frame codec penalty；让 forced EOS 被 `min_tokens` 再次屏蔽后以任意 codec id 继续解码；让无 EOS 的离线请求占满剩余上下文；用本次 simplex 修复改变 native duplex chunk 语义。 +- 验收：覆盖正负 logits、空 history、超过 16 frame 的遗忘、`no_penalties`、逐请求 penalty 与单次消费；覆盖 forced/unforced EOS、`min_tokens`、2048 上限和剩余 context clamp，并用真实长文本 TTS 检查请求在预算边界释放且无静默尾部。^[PR #6458] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index fd3389ab3..4766cff9f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ faa12af4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 42059f19` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7ca2afe5a..1b4f23a8b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ faa12af4` 验证存在 +- 源码校验：以上路径均已在 `main @ 42059f19` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4d1e54572..adf467bfd 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ faa12af4`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 42059f19`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `4882a36c` 四个 inventory 均无变化；本次变更不触及任何 component 或 model owner。 +本次相对 `faa12af4` 本轮记录 MiniCPM-o 4.5 Talker 的 16-frame codec penalty、forced EOS 恢复、离线预算上限及对应 deploy 默认值。模型规则页已达硬限制，仅补充 PR 来源；不能据此推断其他模型或 native duplex 语义已改变。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 4882a36c \ - --to faa12af4 \ + --from faa12af4 \ + --to 42059f19 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 3d6a12616..3d4559cd9 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458"] confidence: high --- From 702dc13f99914c36e9c3730555cbacb854645087 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:46:22 +0800 Subject: [PATCH 258/465] knowledge(vllm-omni): sync 37812146 (PR #6138) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 37812146759975f38247488847e346afe0cab5ee Upstream-PR: #6138 Knowledge-Delta: 新增 TTS 静态能力与运行时 voice 的 ownership 规则，并固化预计算 speaker profile 的模型回调校验合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-loader-contract.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 7 +++++++ knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 28 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d2de4cada..922c967d0 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: faa12af454335fc5627f8143760b7ab70e41e875 + previous_audited_sha: 42059f19d5dc981eec7ba95085bd5380beb85d66 audited_ref: main - audited_sha: 42059f19d5dc981eec7ba95085bd5380beb85d66 + audited_sha: 37812146759975f38247488847e346afe0cab5ee # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0f9383ea9..6befcc36e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -42059f19d5dc981eec7ba95085bd5380beb85d66`. The canonical machine baseline +37812146759975f38247488847e346afe0cab5ee`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 65b48359a..ab5a97e09 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 42059f19` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 37812146` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 633febbe6..b346ae43b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 42059f19` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 37812146` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1c94d03f6..4f7e9f8ed 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 42059f19` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 37812146` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8d6a72acc..f8bad239a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 42059f19` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 37812146` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md index 8543cbb98..eed7917fe 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py"] +sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py", "PR #6138"] confidence: high --- @@ -71,3 +71,10 @@ confidence: high - 禁止：用逐 tensor `default_weight_loader` 绕过 fused assembler；以命中一个 source shard 或 target name 代表整组完整；保留已删除的 `q_proj`/`k_proj`/`v_proj` 属性、让 body-direct 或 nested shards 绕过 wrapper 归一化，或对 packed `[q,k,v]`/`[gate,up]` layout 宣告泛化 colwise TP。 - 验收：用非等 q/KV width 数值核对 `cat([q,k,v])`、`cat([gate,up])` 及 bias，覆盖部分/整组缺 shard；覆盖 current、body-direct、nested 前缀和跨 shard incremental loading；断言 MOSS returned names、Qwen wrapper consumer、NPU mocked `prepare_qkv_weights`/forward 均走 fused 属性，并确认无 TP plan。 ^[PR #5791] +## EXEC-2g — 预计算 speaker profile 必须由模型回调完成校验 + +- 触发：共享 speaker profile 载入或校验逻辑新增模型专属字段、尺寸约束或新的 TTS consumer。 +- 强制：`load_validated_profile_tensors` 只负责 tensor 读取与 `expected_model_type` 校验，并必须接收 `validate_profile(profile, tensors)` 回调；Qwen3-TTS 由 talker hidden size 提供 embedding 维度，VoxCPM2 由各自 validator 提供 profile 合同，adapter 与模型运行时复用同一校验器。 +- 禁止：在共享 helper 内按 model type 分支选择校验、恢复可选的模型专属尺寸参数，或让未通过模型校验的 profile 进入 capability 列表和 prompt 路径。 +- 验收：分别用 Qwen3-TTS 与 VoxCPM2 的合法、错误尺寸、错误元数据和错误 tensor profile 测试回调结果；确认 adapter 载入与模型运行时载入只保留通过校验的 profile，并覆盖 validator 错误时的跳过行为。^[PR #6138] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4766cff9f..d4cc6fa22 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 42059f19` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 37812146` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1b4f23a8b..899416531 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 42059f19` 验证存在 +- 源码校验：以上路径均已在 `main @ 37812146` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 31256b506..5750b0ad4 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -128,6 +128,13 @@ confidence: high - 禁止：只在 `max_tokens=None` 时处理 `min_tokens`；让 `check_stop` 因过大的最小 token 数永远早退；用静默 fallback 掩盖上下文已满。 - 验收：覆盖剩余 46、50、0 token 的请求，断言最终 request 参数、原始参数不变，并回归普通 stage 的默认 max-token 行为。 ^[PR #6346] +### SERV-5l — TTS 静态能力与运行时 voice 必须分属 owner + +- 触发：TTS serving 提取或迁移 built-in speakers、precomputed profiles、supported languages、codec frame rate，或修改 voice upload/list/delete 与 embedding 校验。 +- 强制：adapter 初始化时一次性加载 `TTSCapabilities` 快照；静态模型能力归 adapter，`supported_speakers` 与 `supported_languages` 使用 `frozenset`，`precomputed_speakers` 保存 profile metadata，server 只拥有 `uploaded_speakers`，可用 speakers 由两者并集派生且上传集合独立维护。 +- 禁止：让 server 保存或修改静态能力；把上传 voice 合并进 built-in speaker 集合后在删除时 `discard`，或把 frozen dataclass 误解为内部 profile mapping 的深层不可变；在共享 serving 中恢复 model-type capability 分支。 +- 验收：构造相关 adapter，断言能力在初始化后进入 snapshot、集合类型正确且内置/预计算/上传 voice 的并集完整；删除与 built-in 同名的上传 voice 后内置 voice 仍存在，并验证无 adapter 的 diffusion 路径不会触发 adapter 属性错误或专属维度校验。^[PR #6138] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index adf467bfd..59faa0a92 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 42059f19`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 37812146`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `faa12af4` 本轮记录 MiniCPM-o 4.5 Talker 的 16-frame codec penalty、forced EOS 恢复、离线预算上限及对应 deploy 默认值。模型规则页已达硬限制，仅补充 PR 来源；不能据此推断其他模型或 native duplex 语义已改变。 +本次相对 `42059f19` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from faa12af4 \ - --to 42059f19 \ + --from 42059f19 \ + --to 37812146 \ --repo \ --mode report-only ``` From 05d14593d8cbf93b224763d10ee7b3011707d18b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:48:53 +0800 Subject: [PATCH 259/465] knowledge(vllm-omni): sync 493fc643 (PR #6445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 493fc6439755101c018e5950a4df39f188aaebdb Upstream-PR: #6445 Knowledge-Delta: 新增 MiniMax H3 对 final-layout artifact 的 dtype-neutral model ABI、混合 BF16/FP32 ownership 与 post-restore validator 门禁。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../repos/vllm-omni/models/minimax-h3/rules-loading.md | 9 ++++++++- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 922c967d0..479730cc7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 42059f19d5dc981eec7ba95085bd5380beb85d66 + previous_audited_sha: 37812146759975f38247488847e346afe0cab5ee audited_ref: main - audited_sha: 37812146759975f38247488847e346afe0cab5ee + audited_sha: 493fc6439755101c018e5950a4df39f188aaebdb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6befcc36e..46948b23d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -37812146759975f38247488847e346afe0cab5ee`. The canonical machine baseline +493fc6439755101c018e5950a4df39f188aaebdb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ab5a97e09..0b3c3b02b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 37812146` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 493fc643` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b346ae43b..62315c8a0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 37812146` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 493fc643` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4f7e9f8ed..6a8cb3ca0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 37812146` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 493fc643` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f8bad239a..d2a10220f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 37812146` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 493fc643` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d4cc6fa22..186416de4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 37812146` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 493fc643` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 899416531..20096b941 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 37812146` 验证存在 +- 源码校验：以上路径均已在 `main @ 493fc643` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 59faa0a92..2cebb724a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 37812146`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 493fc643`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `42059f19` 四个 inventory 均无变化；本次变更不改变 runtime registry 或支持范围。 +本次相对 `37812146` 本轮新增 MiniMax H3 的模型级 final-layout restore contract 规则；共享来源身份、表示无关 identity 与精确恢复机制已由 DIFF-2d 统一承载。该规则不能推断 loader 激活、DLO 传输或性能支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 42059f19 \ - --to 37812146 \ + --from 37812146 \ + --to 493fc643 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md index 4c68cb58e..a784240da 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #5910", "PR #6213"] +sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #5910", "PR #6213", "PR #6445"] confidence: high --- @@ -61,3 +61,10 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection - 禁止：用 `_supports_mmap_loading` 等 pipeline capability flag 或 parameter-attached transform 代替 adapter proof；让 backend 重扫 checkpoint、重建 runtime name 或绕过 H3 active loader；把 TP>1 ordinary-loader fallback 描述成 shared-mmap host-memory optimization。 - 验收：adapter sentinel 验证 grouped-QKV 输出顺序且目标 parameter 不携带 mmap transform；preflight 覆盖 TP/HSDP/online quantization、缺 key、shape/dtype mismatch 与未适配 custom loader，并断言回退发生在 mutation 前；exact plan transfer 与 direct-mmap/ordinary-loader 两条路径分别完成 H3 权重完整性检查。^[PR #6213] +## MMH3-1n — H3 final-layout restore 必须由 dtype-neutral model contract 校验混合精度 + +- 触发：MiniMax H3 参与 final-layout host-weight artifact 的 identity/restore，或修改 `MiniMaxH3DiTModel` 构造后的 dtype invariants。 +- 强制：`MiniMaxH3DiTModel` 必须声明 dtype-neutral 的 `FinalLayoutModelContract`，固定 schema、`implementation_id` 与 version，并提供 `validate_restored_host_weights()` 在 exact tensor restore 提交后重新执行 H3 的混合精度不变量校验；BF16 只属于具体 artifact policy/producer，模型 contract 不得绑定 BF16 表示。 +- 禁止：用 dtype 字符串代替版本化 model ABI；遗漏 persistent buffer 或模型要求保留的 FP32 parameter/buffer；把 contract 声明或 validator 存在误认为 loader 已启用、DLO 已接入或 artifact 已具备跨拓扑性能保证。 +- 验收：contract test 断言 `vllm-omni.diffusion.final-layout-tensors-v1`、`minimax-h3-dit`、版本与 validator；reduced MiniMax H3 真实 ownership 同时覆盖 BF16 tensor、模型要求的 FP32 tensor 和 persistent buffer，并在 one-shot restore commit 后执行 validator；另以 exact identity、tensor coverage 与 source-change guard 覆盖失败时不变更模型。^[PR #6445] + From 0f25618ad1c17b9cf15eb6789ac20997f8a1f6c4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:52:42 +0800 Subject: [PATCH 260/465] knowledge(vllm-omni): sync c2daa471 (PR #6073) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: c2daa471c330901e537b6a176dbbce014dbcb307 Upstream-PR: #6073 Knowledge-Delta: 新增共享 diffusion regional compile 的 HookRegistry/DLO dispatcher 边界规则，要求只编译原始 block callable 并同步更新 hook 引用。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 10 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 479730cc7..f07aa2e2f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 37812146759975f38247488847e346afe0cab5ee + previous_audited_sha: 493fc6439755101c018e5950a4df39f188aaebdb audited_ref: main - audited_sha: 493fc6439755101c018e5950a4df39f188aaebdb + audited_sha: c2daa471c330901e537b6a176dbbce014dbcb307 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 46948b23d..644ebb753 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -493fc6439755101c018e5950a4df39f188aaebdb`. The canonical machine baseline +c2daa471c330901e537b6a176dbbce014dbcb307`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0b3c3b02b..3c5900f3f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 493fc643` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c2daa471` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 62315c8a0..d99450254 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 493fc643` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c2daa471` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 9b5bcdab5..4108b86f6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073"] confidence: high --- @@ -214,3 +214,10 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 强制：预处理必须为影响物理 batch 兼容性的 pipeline condition 写入 `batch_compatibility_key`；`RequestScheduler` 必须将显式 `sample_solver` 规范化为去空格小写、将 `flow_shift` 转为 `float`，并保留未指定值为 `None`，再与现有 batch-wide sampling key 一起决定 admission。pipeline 只能批处理兼容 key 相同的请求，按 `num_outputs_per_prompt` 合并 prompt/embedding、latents、request-local generators 和条件，并拆回每个原始请求一个 `DiffusionOutput`；S2V 等 request-local metadata 不得为输出数重复物化。 - 禁止：只按原始 `extra_args`、第一条请求或 pipeline 名称决定兼容性；把不兼容的 image/last-image、VACE 或 S2V 条件混入同一 batch；共享全局 RNG；返回无法按请求对应的聚合输出；把请求隔离和 output split 误当成不同物理 batch shape 的数值等价。 - 验收：scheduler 测试必须区分不同 `sample_solver`、`flow_shift` 和 condition key，并证明等价显式值可归一化合批；Wan2.2 T2V/I2V/VACE/S2V 测试覆盖 generators、latents、prompt embeddings、output split 以及 image shape、last-image presence、VACE condition/reference count、S2V `init_first_frame`/`num_repeat`/audio time length/raw shape/sample rate 的拒绝路径。真实验证还必须分别检查同批请求的身份、prompt、条件、顺序和请求级 RNG 隔离；Wan2.2 BF16 的 batch-1 与 batch-2 轨迹可能不同，未经固定精度策略和 exact case 验证不得声称 bitwise 或生产级等价。^[PR #5676] + +### DIFF-10a — regional compile 必须把 HookRegistry dispatcher 留在图外 + +- 触发：修改 regional `torch.compile`、DLO/普通 layerwise offloader 的 `HookRegistry` wrapper、重复 block 编译或依赖原始 callable 的 hook（如 MagCache）。 +- 强制：发现 `submod._omni_original_forward` 时只编译该原始 compute callable；保留 `submod.forward` 作为 hook dispatcher，将 `_omni_original_forward` 重绑定到 compiled callable，并更新仍指向编译前 callable 的 hook `fn_ref.original_forward`。 +- 禁止：把 `HookRegistry` dispatcher 或其 stream/event/storage/offload 控制流编译进 regional graph；直接覆盖 `forward`；让 hook 保留过期的 eager callable。 +- 验收：测试断言 `torch.compile` 收到原始 callable、`forward` 仍为 wrapper、hook reference 指向 compiled callable，且 block 调用使用编译结果；DLO+regional compile smoke 还须覆盖有限输出与首次 lazy denoise 不崩溃。^[PR #6073] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6a8cb3ca0..09d1f8720 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 493fc643` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c2daa471` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index d2a10220f..492a72e81 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 493fc643` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c2daa471` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 186416de4..4d8d86f68 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 493fc643` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c2daa471` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 20096b941..1a1631313 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 493fc643` 验证存在 +- 源码校验：以上路径均已在 `main @ c2daa471` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2cebb724a..89f3f3ee7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 493fc643`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c2daa471`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `37812146` 本轮新增 MiniMax H3 的模型级 final-layout restore contract 规则；共享来源身份、表示无关 identity 与精确恢复机制已由 DIFF-2d 统一承载。该规则不能推断 loader 激活、DLO 传输或性能支持。 +本次相对 `493fc643` 本轮将 PR #6073 的共享 regional compile 与 HookRegistry/DLO 边界沉淀为 `DIFF-10a`：只编译原始 block callable，并保留 hook dispatcher 在编译图外。不可据此推断 DreamZero 的同类 `setup_compile` 路径、其他模型或硬件上的完整 parity 已修复或验证。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 37812146 \ - --to 493fc643 \ + --from 493fc643 \ + --to c2daa471 \ --repo \ --mode report-only ``` From ad07a77f6356817be2dd042a2f4b6d87ea880965 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 09:56:20 +0800 Subject: [PATCH 261/465] knowledge(vllm-omni): sync 72ee535f (PR #6102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 72ee535fcde303a4802b9b2e9663fad23d6f7cdb Upstream-PR: #6102 Knowledge-Delta: 沉淀 PR #6102 的 paged_scheduler 配置、Worker 物理 KV 生命周期、HunyuanImage3 压缩 GQA 和 CUDA/NPU 平台 hook 合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-system-runtime.md | 7 +++++++ .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- .../components/model-executor/rules-platform-backends.md | 9 ++++++++- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md | 9 ++++++++- 13 files changed, 44 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f07aa2e2f..68d30f058 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 493fc6439755101c018e5950a4df39f188aaebdb + previous_audited_sha: c2daa471c330901e537b6a176dbbce014dbcb307 audited_ref: main - audited_sha: c2daa471c330901e537b6a176dbbce014dbcb307 + audited_sha: 72ee535fcde303a4802b9b2e9663fad23d6f7cdb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 644ebb753..5eb596f75 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c2daa471c330901e537b6a176dbbce014dbcb307`. The canonical machine baseline +72ee535fcde303a4802b9b2e9663fad23d6f7cdb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3c5900f3f..67ab2d4fb 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c2daa471` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 72ee535f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 973f4633b..52133223a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102"] --- # vLLM-Omni 配置开发门禁 @@ -138,6 +138,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：让 `dense_legacy` 因这些字段改变既有 pipeline parallel、cache 或 scheduler 配置；用 truthiness 把 `0` 误判为非法或显式 KV pin；在末端静默丢弃非默认值，或把文本 encoder 的长度字段未经模型语义确认当作 DiT KV 上限。 - 验收：structured、legacy 和 direct 配置路径覆盖默认、非默认、非法值与显式 `0`；paged 路径断言字段进入 native cache/scheduler consumer，`-1` 触发 auto-fit；dense 路径断言原有并行、显存和 token budget 保持不变。 ^[PR #6094] +### CONF-2c — paged_scheduler 的请求行上限必须是正整数且贯穿配置投影 + +- 触发：启用 `paged_scheduler`，或修改 diffusion KV 的请求行容量、native metadata builder 和 BlockTables 初始化。 +- 强制：`diffusion_kv_max_rows_per_request` 必须是严格的正整数，并在 `OmniDiffusionConfig` 与 `_DiffusionConfigProjection` 中统一校验；`paged_scheduler` 必须显式设置该字段，Worker 用它把一个 public request 的 sequence/context rows 容量传入 native cache 初始化。 +- 禁止：接受缺失、`0`、负数、`bool` 或浮点行上限；用 truthiness 合并吞掉显式值；让 `dense_legacy` 因该字段创建 native paged KV，或把 CUDA 平台限制写进通用配置校验。 +- 验收：direct、structured 与 deploy/config projection 分别覆盖缺省、正值和非法值，断言 paged mode 的最终 Worker/native consumer 读取行上限，dense mode 保持原有行为，并覆盖平台无关的配置构造。^[PR #6102] + ### CONF-3a — 争议以展开后的最终配置为准 - 触发：CLI、deploy YAML、`base_config` overlay、platform 覆盖、per-stage override diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d99450254..e560adbc8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c2daa471` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 72ee535f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index cc4ab11e3..a87ae08c9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -177,3 +177,10 @@ confidence: high - 禁止：把任意 `qwen3_vl` 配置归类为 HiDream-O1；给 raw index key 强加 `model.` 前缀；只增加显式 `model_class_name` 映射而让通用 text-to-image/server 自动解析仍落到未注册 architecture；缺失或格式异常的 index 不得误报命中。 - 验收：覆盖 HiDream signature 命中、普通 Qwen3-VL、缺失/错误 `weight_map`、显式类名与 signature 不一致，以及 resolver/enrichment 两条路径；再断言 registry pipeline 和 module-level post-process 均可解析。^[PR #5194] +## DIFF-4u — Worker 物理 paged KV 必须与 Scheduler 逻辑分配分层并原子安装 + +- 触发：启用或修改 diffusion Worker 的 native paged KV data plane、请求 row registry、paged attention adapter、BlockTables 更新或唤醒/终止清理。 +- 强制：Scheduler 只拥有 logical block allocation、generation 和 `DiffusionKVMetadata`；每个 Worker rank 消费自己的 `KVCacheConfig`，拥有物理 KV tensors、native `BlockTables` 和 request/sequence/context rows。安装 metadata 前必须校验 group/count/range/capacity，并以 staged append、apply 和失败 rollback 原子发布；重复 generation 幂等，冲突或过期快照拒绝，BlockTables 变更使已准备的 attention batch 失效，终止时先清 Worker rows 而不释放 Scheduler blocks。 +- 禁止：Worker 自行分配或释放 Scheduler-owned logical blocks；把部分 row 安装、native append/apply 异常或 stale snapshot 当作成功；从 padding 猜异构 prefix 的逻辑布局；在清理或 wake refresh 后继续复用旧的 native metadata/buffer view。 +- 验收：CPU/mock 覆盖 rank-local config、请求/上下文 row mapping、重复/过期/冲突 generation、非法 block 与容量、append/apply rollback、幂等清理和 wake refresh；GPU 另验证非连续 BlockTables、GQA、异构 prefix 与 piecewise attention。该 PR 不证明跨 stage KV transport、W2 prefix-cache 或真实 HunyuanImage3 E2E。^[PR #6102] + diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 09d1f8720..6e971b11e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c2daa471` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 72ee535f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 492a72e81..da2b3f5ad 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c2daa471` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 72ee535f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index 5de367371..a7777ebf9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350"] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102"] confidence: high --- @@ -77,3 +77,10 @@ confidence: high - 禁止：依赖 `NPUWorker` 初始化或 import side effect 建立 diffusion 注册；无条件注册完整 `REGISTERED_ASCEND_OPS` 表；重复调用触发注册断言，或让 `PluggableLayer` 回退到不兼容的上游实现。 - 验收：从真实 NPU diffusion runtime 覆盖首次和重复初始化，断言两个 OOT 名称均解析到可接受 `n_shared_experts` 的实现并完成 HunyuanImage3 fused-MoE accuracy/performance smoke；同时确认普通 NPU、非 NPU 和已完成 vllm-ascend 注册的路径不变。^[PR #6350] +## EXEC-13e — diffusion paged KV 必须经平台 native hook 闭合 + +- 触发：平台为 diffusion paged KV 增加或修改 BlockTables、native attention metadata builder，或 common adapter 需要支持 CUDA 与 NPU 的不同 native contract。 +- 强制：common `OmniPlatform` 通过 `get_diffusion_kv_block_tables_cls()` 与 `build_diffusion_kv_attn_metadata()` 提供默认 native 实现；NPU 必须返回 `AscendBlockTables`，把 `seq_lens_cpu` 转为 `seq_lens_np`，并使用 `AscendAttentionState.ChunkedPrefill`。common diffusion data plane 只调用 platform hooks，并在 layer registration 阶段验证 resolved backend 与 hook 能力。 +- 禁止：在 common adapter 直接导入 `vllm_ascend` 或硬编码 CUDA `BlockTables`/metadata 假设；把 platform capability 延迟到首次 attention dispatch；平台未提供 native contract 时静默回退 dense 或继续占用 paged KV。 +- 验收：default 与 NPU hook 的 class、metadata 参数和 backend selection 通过 mock/contract 测试，unsupported hook/backend 在物理 cache allocation 前 fail fast；CUDA 与 NPU lane 分别验证 native metadata construction，未运行真实 NPU 硬件时不得宣称 Ascend 生产 parity。^[PR #6102] + diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4d8d86f68..6f9771365 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c2daa471` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 72ee535f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1a1631313..2023b9110 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c2daa471` 验证存在 +- 源码校验：以上路径均已在 `main @ 72ee535f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 89f3f3ee7..b02128ff1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c2daa471`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 72ee535f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `493fc643` 本轮将 PR #6073 的共享 regional compile 与 HookRegistry/DLO 边界沉淀为 `DIFF-10a`：只编译原始 block callable，并保留 hook dispatcher 在编译图外。不可据此推断 DreamZero 的同类 `setup_compile` 路径、其他模型或硬件上的完整 parity 已修复或验证。 +本次相对 `c2daa471` 本轮新增配置、共享 diffusion、HunyuanImage3 与 model-executor 平台规则，分别记录 paged_scheduler 行上限、Worker 物理页与 Scheduler 逻辑分配的边界、H3 GQA 压缩以及 CUDA/NPU platform hook。不能由这些规则推断跨 stage KV transport、W2 prefix cache、真实 HunyuanImage3 E2E、NPU 硬件 parity 或一般模型/平台生产支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 493fc643 \ - --to c2daa471 \ + --from c2daa471 \ + --to 72ee535f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md index 84522f23d..3ad025f47 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-13 updated: 2026-09-04 type: rule tags: [vllm-omni, models, hunyuan-image3] -sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", "PR #6306"] +sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", "PR #6306", "PR #6102"] --- # HunyuanImage3 开发规则 @@ -141,3 +141,10 @@ sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/dif - 强制：由 `current_omni_platform.is_cuda()` 选择 `nvidia` 实现，其他后端保留 `native` 实现；两套 block 必须保持相同的公开构造与 checkpoint `state_dict` key 布局，NVIDIA 版本只能通过共享融合算子获得加速；高分辨率 VAE 必须由统一 VAE tiling 管理，任何允许 tiling 关闭的入口都要显式定义容量、回退或拒绝策略；`torch.backends.cudnn` 这类进程级标志应在 worker 边界配置，若保留 block 内临时设置则必须证明 forward 不会并发。 - 禁止：在非 CUDA 平台导入或强制执行 NVIDIA block；把 Triton 可导入等同于所有平台支持；改变 `in_layers`/`out_layers` 等 checkpoint-facing module key；移除逐卷积内存保护后仍让 tiling-disabled 的高分辨率请求无条件执行；或在可能重叠的 forward 中修改进程级 cuDNN 标志。 - 验收：分别以 native/eager 与 CUDA/NVIDIA 实现覆盖 VAE、DiT 的真实 shape、dtype 和输出误差；验证旧 checkpoint 的 `state_dict` 加载与 key parity；对启用 tiling、关闭 tiling及直接 VAE 入口执行高分辨率容量/失败行为测试，并在并发 worker 或明确串行证明下验证 cuDNN 配置与恢复；性能数字只在固定模型、硬件、并行拓扑和 workload 下复测。^[PR #6306] + +- **HY3-5h — paged GQA 必须保留压缩 K/V，dense 路径保持扩展** + - 触发：修改 HunyuanImage3 的 `ImageKVCacheManager`、Q/K/V head layout，或在 dense 与 Scheduler-managed paged attention 之间切换 GQA 处理。 + - 强制：HunyuanImage3 的 dense 路径继续把 K/V 从 8 个 KV heads 扩展到 32 个 Q heads；paged KV active 时必须保留原生压缩的 32Q/8KV 形状，由 native adapter/backend 消费 `num_heads=32` 与 `num_kv_heads=8`。 + - 禁止：在 paged adapter 中把已经压缩的 K/V 再扩成 Q heads，或为兼容 generic shape 在 adapter 内猜测并折叠重复 head；也不得让 paged 分支改变既有 dense 路径的输出布局。 + - 验收：以实际 32Q/8KV geometry 覆盖 dense 与 paged 两条路径，断言 attention consumer 的 K/V shape 分别为 32 和 8，并覆盖 adapter 的 packed Q/K/V 与 native backend 调用；真实模型数值和生产 E2E 仍需独立验证。^[PR #6102] + From 69d3a84682c2ef47496e1d7088dc0bb72ddc099e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 10:02:47 +0800 Subject: [PATCH 262/465] knowledge(vllm-omni): sync b39f599b (PR #6308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b39f599b09a9ec06cfef05213273de7ed6626b7f Upstream-PR: #6308 Knowledge-Delta: 新增单 GPU diffusion 进程内 executor 的选择、故障闭锁、健康检查与关闭合同，并固定 omitted backend 由运行时按 GPU 数解析。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/configuration/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-output-lifecycle.md | 9 ++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 68d30f058..f713f0c1c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c2daa471c330901e537b6a176dbbce014dbcb307 + previous_audited_sha: 72ee535fcde303a4802b9b2e9663fad23d6f7cdb audited_ref: main - audited_sha: 72ee535fcde303a4802b9b2e9663fad23d6f7cdb + audited_sha: b39f599b09a9ec06cfef05213273de7ed6626b7f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5eb596f75..d8c7790cb 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -72ee535fcde303a4802b9b2e9663fad23d6f7cdb`. The canonical machine baseline +b39f599b09a9ec06cfef05213273de7ed6626b7f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 67ab2d4fb..2b32b6726 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 72ee535f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b39f599b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 52133223a..f125295bb 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308"] --- # vLLM-Omni 配置开发门禁 @@ -111,6 +111,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：用未注册 env var 或模型初始化时自行读取绕过配置 owner；只在 YAML/dataclass 声明但在 engine args 或最终 stage 丢失；把模型专用字段变成所有 talker 的通用行为，或把 absent 当成显式启用。 - 验收：用 direct、structured、legacy 入口的非默认值追到最终 stage config 和真实模型 consumer；断言默认值保持关闭、其他模型不产生副作用，并覆盖字段集合与 projection parity。^[PR #5048] +### VOMNI-CFG-1n — diffusion backend 的 omitted 语义必须留给运行时解析 + +- 触发：修改 `distributed_executor_backend` 的默认值、配置 projection、Async stage 参数透传或 diffusion executor 选择逻辑。 +- 强制：省略值必须以 `None` 贯穿 `OmniDiffusionConfig`、`_DiffusionConfigProjection` 和 async stage config，由 `DiffusionExecutor.get_class()` 按 `num_gpus` 解析为单 GPU 的 `uni`、多 GPU 的 `mp`；显式 `mp` 或 `uni` 必须保持调用方选择。 +- 禁止：在配置 projection 或 `AsyncOmniEngine` 中把省略值预先改成 `"mp"`，把 `None` 当作显式 backend，或用 truthiness 合并吞掉显式 backend；不能以文档或原始 YAML 值代替最终 stage config 的传播证明。 +- 验收：direct、structured 和 async 路径均断言 omitted 值在最终 stage config 仍为 `None`；覆盖单 GPU→`UniProcDiffusionExecutor`、多 GPU→`MultiprocDiffusionExecutor`、显式 `mp` 和显式 `uni`，并确认已 pin `mp` 的 deploy 配置不变。 ^[PR #6308] + ### CONF-1a — 多 stage 共卡时 diffusion stage 必须显式设 gpu_memory_utilization - 触发：多 stage 模型 stage 共卡时，分布式测试在模型加载期 CUDA OOM。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e560adbc8..b7dfbb754 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 72ee535f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b39f599b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 0f97b54cb..168a20fd6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6288"] +sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6288", "PR #6308"] confidence: high --- @@ -115,3 +115,10 @@ confidence: high - 强制：`mux_av_video_audio_bytes()` 必须在 context manager 内打开并配置 MP4 video/audio streams，消费 `Iterable[av.VideoFrame]` 后 flush video encoder；音频按 `fltp` 和 mono/stereo layout 构造，并以 `pts=0`、`time_base=1/sample_rate` 标记输入时间零点。成功和 generator、encoder、mux 异常都必须退出 context 关闭 container，同时原样传播原始错误。 - 禁止：在 generator 消费前或 context 外打开/使用 container；只在成功路径手工 close；吞掉帧生成或编码异常后静默改走其他 muxer；把 mock container cleanup 或单一 CPU 运行误认为完整跨硬件 A/V parity。 - 验收：测试覆盖预构造 `gbrp` 帧的 H.264 mux、无音频/mono/stereo 音频、AAC 首个有效 packet 的负 priming PTS、video flush 和 container close；让 frame generator 在中途抛错并断言 container 已关闭且异常 identity 保持，再用固定媒体输入执行 ffprobe 与完整 FFmpeg 解码。^[PR #6288] + +## DIFF-11a — 单 GPU diffusion executor 必须保持进程内故障与生命周期语义 + +- 触发：新增或修改单 GPU diffusion executor、inline worker RPC、设备故障处理或 executor shutdown 生命周期。 +- 强制：单 GPU 的 `uni` 路径必须在 engine 进程内构造并驱动 `WorkerWrapperBase`，仍初始化一个 rank 的 distributed environment，但不得创建 worker subprocess、`MessageQueue` 或 IPC 输出段；`collective_rpc` 直接调用 worker，保留 reply-rank 的返回形状，recoverable request error 留在请求级，sticky accelerator fault 必须 latch executor failure、只触发一次 failure callback，并让 `check_health()` 抛出 `EngineDeadError`。 +- 禁止：单 GPU 默认继续走 `mp`，多 GPU 选择 `uni`，把 inline `timeout` 描述成可强制的 RPC deadline，或把所有 worker exception 都升级为 engine death；设备上下文已中毒后不得继续报告健康，也不得让部分构造失败或重复 shutdown 遗留 worker、callback、缓存或 device allocation。 +- 验收：覆盖 omitted/显式 backend/多 GPU 的选择矩阵、直接 RPC 与返回形状、recoverable error、CUDA/NPU sticky fault、callback once、health failure、部分构造 cleanup 和幂等 shutdown；真实单 GPU smoke 还须验证一 rank group、`InlineStageDiffusionClient`、真实 `DiffusionOutput` 流程、worker shutdown，以及固定 seed 下与 pinned `mp` 的输出一致性，mock 不能替代这些证据。 ^[PR #6308] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6e971b11e..e0145062f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 72ee535f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b39f599b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index da2b3f5ad..74d3181a3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 72ee535f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b39f599b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6f9771365..333713a8b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 72ee535f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b39f599b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2023b9110..51e16b21d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 72ee535f` 验证存在 +- 源码校验：以上路径均已在 `main @ b39f599b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b02128ff1..552731f45 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 72ee535f`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b39f599b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,7 +45,7 @@ OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusio `minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `c2daa471` 本轮新增配置、共享 diffusion、HunyuanImage3 与 model-executor 平台规则，分别记录 paged_scheduler 行上限、Worker 物理页与 Scheduler 逻辑分配的边界、H3 GQA 压缩以及 CUDA/NPU platform hook。不能由这些规则推断跨 stage KV transport、W2 prefix cache、真实 HunyuanImage3 E2E、NPU 硬件 parity 或一般模型/平台生产支持。 +本次相对 `72ee535f` 本轮新增两条可复用合同：diffusion 单 GPU 的进程内 executor 生命周期，以及 omitted `distributed_executor_backend` 必须保留到运行时按设备数解析。文档、测试改动和 serving 中仅承担该配置透传的实现细节未单独沉淀为规则，也不能据此推断所有硬件上的性能或生产级支持。 ## AR/omni 模型族（29） @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from c2daa471 \ - --to 72ee535f \ + --from 72ee535f \ + --to b39f599b \ --repo \ --mode report-only ``` From 013d199aadfe08c33daca76a6035fb4cad527fba Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 11:00:13 +0800 Subject: [PATCH 263/465] knowledge(vllm-omni): sync f09bdcbc (PR #6255) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: f09bdcbc7dc94827e76d4e9479674639be6ac442 Upstream-PR: #6255 Knowledge-Delta: 更新 async-output wait 的可配置正数超时、无效值回退、双路径一致性与诊断合同，并推进 release baseline 和源码 pin。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 4 ++-- .../components/diffusion/rules-output-lifecycle.md | 8 ++++---- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 4 ++-- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f713f0c1c..a385969a2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 72ee535fcde303a4802b9b2e9663fad23d6f7cdb + previous_audited_sha: b39f599b09a9ec06cfef05213273de7ed6626b7f audited_ref: main - audited_sha: b39f599b09a9ec06cfef05213273de7ed6626b7f + audited_sha: f09bdcbc7dc94827e76d4e9479674639be6ac442 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d8c7790cb..e5db09260 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b39f599b09a9ec06cfef05213273de7ed6626b7f`. The canonical machine baseline +f09bdcbc7dc94827e76d4e9479674639be6ac442`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2b32b6726..ed90ed152 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b39f599b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f09bdcbc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b7dfbb754..aaa1a56d8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b39f599b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f09bdcbc` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -38,7 +38,7 @@ sources: [] | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | -| video/audio mux、DLO DP wave、result queue、async pump、SHM ownership 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | +| video/audio mux、DLO DP wave、result queue、async pump、SHM ownership、async-output wait 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | request-wave admission coalescing、stable window、deadline 与 finite config | [admission wait rules](rules-admission-wait.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 168a20fd6..a4864e35a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6288", "PR #6308"] +sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308"] confidence: high --- @@ -94,9 +94,9 @@ confidence: high ## DIFF-1q — request-mode async output 交付必须覆盖传输、竞态与生命周期 - 触发：修改 request-mode diffusion 的 async D2H/SHM output、`MessageQueue` 写入、batch split、result pump、future bookkeeping，或 worker 的 sleep/内存释放生命周期。 -- 强制：per-request tensor view 的 IPC packing 必须按 `max(view_bytes, storage_bytes)` 判断并只把 view 内容放入 SHM；`execute_batch` 注册 split map 时必须在同一锁下接管早到的 batch output；`result_mq` 的 `COMPUTE_DONE` 与 `OUTPUT_READY` 写入必须串行化；result pump 必须先完成 SHM unpack，再在 `_futures_lock` 下原子 resolve waiter 或缓存 completed output；worker 执行 `sleep`/`handle_sleep_task` 和本地 sleep task 前必须等待 `drain_async_outputs()`，并在 `_ASYNC_OUTPUT_TIMEOUT` 超时日志中输出 executor bookkeeping。 -- 禁止：按 view 大小判断而让共享 batch storage 被 pickle 重复发送；在 split map 建立前缓存一个无人消费的 batch-level output；让多个线程并发写单写者 `MessageQueue`；SHM unpack 期间释放 future 锁导致 waiter orphan；释放 device memory 时仍允许后台 D2H/SHM 线程读取源 tensor；把单 tensor storage-aware packing 当作 aggregate serialized message 已不会溢出的证明，也不得据此推断外部 HTTP 视频传输已优化。 -- 验收：覆盖 per-request shared-storage wire-size regression、早到和晚到的 batch `OUTPUT_READY`、已有 waiter 与缓存 future 的原子 resolution、并发 result queue writer 无 overlap、pending output 的 drain/timeout 及 sleep-before-drain 顺序；同时验证 non-sleep RPC 不 drain、SHM unpack/error path、超时诊断状态和 worker shutdown cleanup。^[PR #6023] +- 强制：per-request tensor view 的 IPC packing 必须按 `max(view_bytes, storage_bytes)` 判断并只把 view 内容放入 SHM；`execute_batch` 注册 split map 时必须在同一锁下接管早到的 batch output；`result_mq` 的 `COMPUTE_DONE` 与 `OUTPUT_READY` 写入必须串行化；result pump 必须先完成 SHM unpack，再在 `_futures_lock` 下原子 resolve waiter 或缓存 completed output；worker 执行 `sleep`/`handle_sleep_task` 和本地 sleep task 前必须等待 `drain_async_outputs()`。两个 `DiffusionEngine` wait 路径必须逐次解析 `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT`：正数 float 可覆盖 600s 默认值；无效值告警后回退默认值；超时日志必须给出可调的变量名与 executor bookkeeping。 +- 禁止：按 view 大小判断而让共享 batch storage 被 pickle 重复发送；在 split map 建立前缓存一个无人消费的 batch-level output；让多个线程并发写单写者 `MessageQueue`；SHM unpack 期间释放 future 锁导致 waiter orphan；释放 device memory 时仍允许后台 D2H/SHM 线程读取源 tensor；把单 tensor storage-aware packing 当作 aggregate serialized message 已不会溢出的证明，也不得据此推断外部 HTTP 视频传输已优化；不得在 request path 因无效 timeout 环境值抛出异常，也不得只修改其中一个 wait 路径。 +- 验收：覆盖 per-request shared-storage wire-size regression、早到和晚到的 batch `OUTPUT_READY`、已有 waiter 与缓存 future 的原子 resolution、并发 result queue writer 无 overlap、pending output 的 drain/timeout 及 sleep-before-drain 顺序；同时验证 non-sleep RPC 不 drain、SHM unpack/error path、超时诊断状态和 worker shutdown cleanup，以及两个 wait 路径的默认值、正数 integer/float override、逐次重读与非数/非正值的 warning+fallback。^[PR #6023] ^[PR #6255] ## DIFF-1r — 共享 Future resolve 必须对取消竞态 fail-safe diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e0145062f..2445dd671 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b39f599b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f09bdcbc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 74d3181a3..96184e392 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b39f599b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f09bdcbc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 @@ -40,4 +40,4 @@ sources: [] | shared image example envelope 与 model_extras 参数声明 | [image task envelope 合同](rules-image-task-envelope.md) | | Omni 输出类型与字段/复制合同 | [输出类型合同](rules-output-contract.md) | | 采样循环不变量、热路径缓存、AR 音频侧路 | [运行时热路径合同](rules-runtime-hot-paths.md) | -| NPU runner、ROCm 分页注意力、NPU 模型补丁 | [平台后端合同](rules-platform-backends.md) | \ No newline at end of file +| NPU runner、ROCm 分页注意力、NPU 模型补丁 | [平台后端合同](rules-platform-backends.md) | diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 333713a8b..21798e515 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b39f599b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f09bdcbc` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 51e16b21d..c41923345 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b39f599b` 验证存在 +- 源码校验：以上路径均已在 `main @ f09bdcbc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 552731f45..7c52b41fa 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b39f599b`（2026-08-12）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f09bdcbc`（2026-08-22）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 72ee535f \ - --to b39f599b \ + --from b39f599b \ + --to f09bdcbc \ --repo \ --mode report-only ``` From 1b84440c1e9be5a581105431f6ba9582bc7aeef6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 11:05:52 +0800 Subject: [PATCH 264/465] knowledge(vllm-omni): sync bd4f9acf (PR #6182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: bd4f9acfd30456cb8fa98af53d32f7adc34e03a0 Upstream-PR: #6182 Knowledge-Delta: 沉淀 pipeline sampling constraints 的 stage runtime 传播、pipeline-wins 优先级、typed 参数重建与 caller 不可变合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 4 ++-- .../vllm-omni/components/configuration/rules.md | 12 ++++++++++-- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 4 ++-- .../components/serving/rules-request-input.md | 10 ++++++++-- .../repos/vllm-omni/components/serving/rules.md | 3 ++- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 12 files changed, 34 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a385969a2..1c338e9ad 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b39f599b09a9ec06cfef05213273de7ed6626b7f + previous_audited_sha: f09bdcbc7dc94827e76d4e9479674639be6ac442 audited_ref: main - audited_sha: f09bdcbc7dc94827e76d4e9479674639be6ac442 + audited_sha: bd4f9acfd30456cb8fa98af53d32f7adc34e03a0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e5db09260..40090438a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f09bdcbc7dc94827e76d4e9479674639be6ac442`. The canonical machine baseline +bd4f9acfd30456cb8fa98af53d32f7adc34e03a0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ed90ed152..74285b44c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f09bdcbc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bd4f9acf` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 @@ -35,7 +35,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] | 遇到什么 | 查看哪里 | |---|---| | 理解配置从 deploy、CLI、默认 factory 到 structured/legacy config 的稳定边界 | [配置构造架构](architecture.md) | -| 根据 PR 描述直达 strict schema、deploy/topology、composable strategy 或显存配置的规则组与第一批源码 | [配置开发门禁与代码地图](rules.md) | +| 根据 PR 描述直达 strict schema、pipeline sampling constraints、deploy/topology、composable strategy 或显存配置的规则组与第一批源码 | [配置开发门禁与代码地图](rules.md) | | 审计配置来源和多层加工 | [config audit](config-audit-plain-language.md) | | 设计配置归一化与 parity 验证 | [config normalization parity](config-normalization-parity.md) | | 查询 `Omni()` 初始化参数 | [Omni init args](omni-init-args.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index f125295bb..08b1b9ccd 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182"] --- # vLLM-Omni 配置开发门禁 @@ -25,6 +25,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | composable strategy、axis、routing、load balancing、`strategy-config` | `composable-strategy`：`CONF-4a` | `vllm_omni/config/composable_parallel/strategy_loader.py::{parse_strategy_specs,load_strategy_specs}` → `translator.py::translate_strategy_stack` → `apply.py::apply_strategy_specs` → `config_factory.py::{StageConfigFactory._apply_strategy_specs,StageConfigFactory._reconcile_strategy_with_cli}` | | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | | multi-stage HF sub-config、stage quantization view、`hf_config_name` | `stage-model-config`：`VOMNI-CFG-1i` | pipeline stage declaration → `OmniModelConfig::{draw_hf_text_config,get_model_arch_config}` → vLLM quantization selection | +| pipeline `sampling_constraints`、`StageConfig.to_omegaconf()`、runtime stage config | `stage-config-propagation`：`VOMNI-CFG-1o` | `merge_pipeline_deploy` → `StageConfig.to_omegaconf()` → `engine.stage_configs` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -34,6 +35,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5g` 见 [topology 与部署 profile](rules-topology-profiles.md) | | `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a`, `CONF-4c`–`4d`，见 [并行拓扑合同](rules-parallel-topology.md) | | `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | +| `stage-config-propagation` | pipeline sampling constraints、StageConfig serialization/runtime config | `VOMNI-CFG-1o` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `VOMNI-CFG-0a`, `VOMNI-CFG-0b` | ## 配置归一化与新老路径一致性 @@ -118,6 +120,13 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：在配置 projection 或 `AsyncOmniEngine` 中把省略值预先改成 `"mp"`，把 `None` 当作显式 backend，或用 truthiness 合并吞掉显式 backend；不能以文档或原始 YAML 值代替最终 stage config 的传播证明。 - 验收：direct、structured 和 async 路径均断言 omitted 值在最终 stage config 仍为 `None`；覆盖单 GPU→`UniProcDiffusionExecutor`、多 GPU→`MultiprocDiffusionExecutor`、显式 `mp` 和显式 `uni`，并确认已 pin `mp` 的 deploy 配置不变。 ^[PR #6308] +### VOMNI-CFG-1o — pipeline sampling constraints 必须穿过 stage runtime 配置 + +- 触发：新增或修改 pipeline 的 `sampling_constraints`、`StageConfig` 字段、deploy 合并或 dataclass→OmegaConf runtime 转换。 +- 强制：每个 stage 的 resolved `sampling_constraints` 必须从 `merge_pipeline_deploy()` 复制到 `StageConfig`，并由 `to_omegaconf()` 导出为可在 `engine.stage_configs` 读取的原生 mapping；不得与默认 sampling params 混为唯一保存位置。 +- 禁止：只在 dataclass 或默认 params 上保留约束、让 OmegaConf runtime config 静默丢字段，或以测试直接注入 constraints list 代替真实序列化路径。 +- 验收：从 pipeline/deploy 合并走到 `StageConfig.to_omegaconf()` 和 runtime extraction，断言每 stage 的非默认 constraint 可读回；再由 serving 侧验证它到最终 sampling consumer 的效果。^[PR #6182] + ### CONF-1a — 多 stage 共卡时 diffusion stage 必须显式设 gpu_memory_utilization - 触发：多 stage 模型 stage 共卡时，分布式测试在模型加载期 CUDA OOM。 @@ -220,4 +229,3 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 强制：将 registry 所需的静态 `PipelineConfig` 放入无 runtime 副作用的轻量模块；registry 只导入该模块，runtime pipeline 继续 re-export 同一对象以保持既有导入路径，并用干净子进程验证 registry 解析不加载 runtime。 - 禁止：让 `pipeline_registry` eager import 模型 runtime、依赖已预加载模块或导入顺序掩盖循环；不得因拓扑模块轻量就改变模型 runtime 的实际加载语义。 - 验收：隔离进程中导入并解析目标 registry key，断言对应 runtime module 不在 `sys.modules` 且返回的 `PipelineConfig` 身份与字段正确；再验证 runtime 的兼容导入路径仍 re-export 同一拓扑对象。^[PR #6322] - diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index aaa1a56d8..55465d488 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f09bdcbc` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bd4f9acf` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2445dd671..10202a9d2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f09bdcbc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bd4f9acf` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 96184e392..7d6df9abe 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f09bdcbc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bd4f9acf` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 21798e515..e3b744fea 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f09bdcbc` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bd4f9acf` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c41923345..d3f10e958 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f09bdcbc` 验证存在 +- 源码校验：以上路径均已在 `main @ bd4f9acf` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -28,7 +28,7 @@ sources: [] | 遇到什么 | 查看哪里 | |---|---| | 理解入口到 engine 的边界 | [architecture](architecture.md) | -| 根据 PR 描述直达 request contract、streaming format、artifact readiness、factory lifecycle 或 metrics 的规则组与第一批源码 | [rules 与代码地图](rules.md) | +| 根据 PR 描述直达 request contract、stage sampling constraints、streaming format、artifact readiness、factory lifecycle 或 metrics 的规则组与第一批源码 | [rules 与代码地图](rules.md) | | upstream launcher shutdown、renderer warmup 与多 replica compile-cache 隔离 | [upstream 兼容规则](rules-upstream-compat.md) | | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md index 9b6779f53..e240c2bb0 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -4,13 +4,13 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3805", "PR #5374", vllm_omni/entrypoints/openai/, vllm_omni/inputs/, "PR #5181"] +sources: ["PR #3805", "PR #5374", vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/inputs/, tests/entrypoints/test_omni_entrypoints.py, "PR #5181", "PR #6182"] confidence: high --- # 请求输入合同 -`SERV-4a`–`SERV-4m`：公开请求字段的校验、限界与 owner。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口。 +`SERV-4a`–`SERV-4o`：公开请求字段的校验、限界与 owner。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口。 ## SERV-4a — 公开字段由 serving 显式拥有 @@ -118,3 +118,9 @@ engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)； - 禁止：用 truthiness 判断 `num_cached_tokens` 而丢弃零值；只在一种响应模式填充详情；丢弃 `multimodal_tokens` 或以不同逻辑分别构造两种 usage；将该响应合同混同为 Prometheus 指标合同。 - 验收：分别覆盖 streaming 与 non-streaming、`cached_tokens=0` 及非零值、image/audio 多模态计数和详情开关关闭场景，断言响应字段一致、计数准确且关闭时不产生详情。^[PR #5181] +## SERV-4o — pipeline sampling constraints 必须在 caller 参数上重建并优先 + +- 触发：修改 `OmniBase.resolve_sampling_params_list()`、stage runtime config、pipeline `sampling_constraints`，或让 caller 提供单/多 stage sampling params。 +- 强制：从每个 runtime stage config 取得约束并与同 stage caller params 合并；pipeline-required key 覆盖 caller 冲突值，其他 caller fields 保留。对 mapping 复制合并；对 dataclass/msgspec sampling object 以合并值重建，使 constructor/post-init 重新计算 stop-token 等 derived state，且 caller/default object 保持不变。 +- 禁止：caller 提供 params 时整包替换 pipeline constraints；只从 dataclass 而不是实际 OmegaConf runtime config 读取；对已构造 `SamplingParams` setattr/copy 后跳过 derived-state 更新，或静默以 caller 值赢得 pipeline-required field。 +- 验收：真实 stage config conversion 覆盖单与多 stage、normal caller field、pipeline-required `detokenize`/stop token conflict、derived stop-token state 及 caller immutability；默认 params 与无约束 control 保持既有行为。^[PR #6182] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 5c200a42d..e7e662450 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -28,6 +28,7 @@ confidence: high | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | | request-level LoRA、AR-only/multistage、streaming input update、stage cardinality | `request-contract`：`SERV-4l` | `serving_chat.py` LoRA resolve → `AsyncOmni.generate` → `AsyncOmniEngine._build_add_request_message` → stage-0 input processor | +| caller sampling params、pipeline `sampling_constraints`、stage config runtime extraction | `request-contract`：`SERV-4o` | `OmniBase.resolve_sampling_params_list` → `_get_sampling_constraints_list` → `_apply_sampling_constraints` | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | | video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | @@ -44,7 +45,7 @@ confidence: high | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | | `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection、replica fault isolation | `SERV-5a`–`SERV-5e`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | -| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view | `SERV-4a`–`4h`, `SERV-4l`–`4m`，全部见 [请求输入合同](rules-request-input.md) | +| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints | `SERV-4a`–`4h`, `SERV-4l`–`4o`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 7c52b41fa..1b006e8c6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f09bdcbc`（2026-08-22）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bd4f9acf`（2026-08-22）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b39f599b \ - --to f09bdcbc \ + --from f09bdcbc \ + --to bd4f9acf \ --repo \ --mode report-only ``` From 0f31b271a588cc3a307a1b25b493699916c804bb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 11:08:56 +0800 Subject: [PATCH 265/465] knowledge(vllm-omni): sync 855719e3 (PR #6385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 855719e3318bcc96ed070a6d331a19532f47c7d0 Upstream-PR: #6385 Knowledge-Delta: 已有 DIFF-4d 完整覆盖 worker 异常 traceback、rank-status 与 device cache 清理合同；本提交推进 release baseline 和源码 pin。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1c338e9ad..1dc6d5c4b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f09bdcbc7dc94827e76d4e9479674639be6ac442 + previous_audited_sha: bd4f9acfd30456cb8fa98af53d32f7adc34e03a0 audited_ref: main - audited_sha: bd4f9acfd30456cb8fa98af53d32f7adc34e03a0 + audited_sha: 855719e3318bcc96ed070a6d331a19532f47c7d0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 40090438a..faa2fabe6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bd4f9acfd30456cb8fa98af53d32f7adc34e03a0`. The canonical machine baseline +855719e3318bcc96ed070a6d331a19532f47c7d0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 74285b44c..25463621d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bd4f9acf` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 855719e3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 55465d488..851b0dd4b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bd4f9acf` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 855719e3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 10202a9d2..5a3210690 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bd4f9acf` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 855719e3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7d6df9abe..2ddadfd4e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bd4f9acf` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 855719e3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e3b744fea..e162f1177 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bd4f9acf` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 855719e3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d3f10e958..37e52d0bf 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bd4f9acf` 验证存在 +- 源码校验：以上路径均已在 `main @ 855719e3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1b006e8c6..cbb1eee47 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bd4f9acf`（2026-08-22）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 855719e3`（2026-08-22）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from f09bdcbc \ - --to bd4f9acf \ + --from bd4f9acf \ + --to 855719e3 \ --repo \ --mode report-only ``` From 7930e90f9d85ee781754bf138a31046e86d8972d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 11:12:38 +0800 Subject: [PATCH 266/465] knowledge(vllm-omni): sync 3ad64c6a (PR #6449) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 3ad64c6a1236dfdeb26354f8e4be72a97a4c1a04 Upstream-PR: #6449 Knowledge-Delta: 新增 Qwen3-Omni CUDA Thinker MRoPE custom-op 边界、平台作用域与固定种子数值/音频一致性合同。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/qwen-omni/_index.md | 1 + .../repos/vllm-omni/models/qwen-omni/rules.md | 18 ++++++++++++++++++ 11 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/qwen-omni/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1dc6d5c4b..e9886b6eb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bd4f9acfd30456cb8fa98af53d32f7adc34e03a0 + previous_audited_sha: 855719e3318bcc96ed070a6d331a19532f47c7d0 audited_ref: main - audited_sha: 855719e3318bcc96ed070a6d331a19532f47c7d0 + audited_sha: 3ad64c6a1236dfdeb26354f8e4be72a97a4c1a04 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index faa2fabe6..53e9bdd80 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -855719e3318bcc96ed070a6d331a19532f47c7d0`. The canonical machine baseline +3ad64c6a1236dfdeb26354f8e4be72a97a4c1a04`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 25463621d..0161080a6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 855719e3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3ad64c6a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 851b0dd4b..015fe21e6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 855719e3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 3ad64c6a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5a3210690..bb4456b9d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 855719e3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3ad64c6a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2ddadfd4e..4870626b8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 855719e3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3ad64c6a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e162f1177..ec43d337f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 855719e3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3ad64c6a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 37e52d0bf..644d9b70e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 855719e3` 验证存在 +- 源码校验：以上路径均已在 `main @ 3ad64c6a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index cbb1eee47..53c3b97a8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 855719e3`（2026-08-22）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3ad64c6a`（2026-08-22）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from bd4f9acf \ - --to 855719e3 \ + --from 855719e3 \ + --to 3ad64c6a \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index 74d71078f..5034be3fb 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -47,3 +47,4 @@ sources: ["PR #5073", "PR #5671", "PR #5976", vllm_omni/model_executor/models/qw | 遇到什么 | 查看哪里 | |---|---| | stage 拓扑、代际差异与官方性能优化结论 | [architecture](architecture.md) | +| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary 或固定种子音频回归 | [Qwen-Omni rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md new file mode 100644 index 000000000..111c71b3d --- /dev/null +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -0,0 +1,18 @@ +--- +title: "Qwen-Omni 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, qwen-omni] +sources: ["PR #6449", vllm_omni/deploy/qwen3_omni_moe.yaml, tests/config/test_config_factory.py] +confidence: high +--- + +# Qwen-Omni 规则 + +## QOMNI-1a — Qwen3-Omni Thinker MRoPE 必须保留 CUDA custom-op 边界 + +- 触发：修改 `qwen3_omni_moe.yaml` 的 CUDA compilation 配置、Thinker MRoPE/RMSNorm 路径，或升级会改变 Inductor MRoPE fusion 的 vLLM 版本。 +- 强制：CUDA platform overlay 仅为 Thinker stage 0 设置 `compilation_config.custom_ops: ["+rotary_embedding"]`，保持 MRoPE 在 CUDA/Triton custom-op 边界后；Talker/Code2Wav 与 CPU、MUSA、NPU、ROCm、XPU 配置不得继承此 override。 +- 禁止：以全局 `enforce_eager` 取代窄 graph boundary；启用更宽的 Q/K norm-RoPE fusion flag；只检查 YAML 文本，或将 Thinker 数值稳定性外推为全部 Qwen stage/平台。 +- 验收：经 `_apply_platform_overrides → merge_pipeline_deploy` 断言 CUDA 仅 stage 0 有该 custom op、其余 stage/平台均无；固定 seed 下与 eager oracle 的 thinker/talker/codec 离散轨迹必须完全一致，并验证生成 WAV 逐字节一致。相关配置展开见 [Configuration rules](../../components/configuration/rules.md)，家族 topology 见 [architecture](architecture.md)。^[PR #6449] From ff9abfeb75184c5b8f06e89654d1eb840741435f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 11:17:39 +0800 Subject: [PATCH 267/465] knowledge(vllm-omni): sync b2c93765 (PR #6468) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: b2c93765393af9d8f8b898db25f958a890ce9ddb Upstream-PR: #6468 Knowledge-Delta: 扩展并行测试基础设施合同，要求 torch.distributed 使用 file rendezvous，并保留真实 TCP 服务的端口 helper。 --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 12 ++++++------ .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e9886b6eb..667db6e5c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 855719e3318bcc96ed070a6d331a19532f47c7d0 + previous_audited_sha: 3ad64c6a1236dfdeb26354f8e4be72a97a4c1a04 audited_ref: main - audited_sha: 3ad64c6a1236dfdeb26354f8e4be72a97a4c1a04 + audited_sha: b2c93765393af9d8f8b898db25f958a890ce9ddb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 53e9bdd80..83a152fc7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3ad64c6a1236dfdeb26354f8e4be72a97a4c1a04`. The canonical machine baseline +b2c93765393af9d8f8b898db25f958a890ce9ddb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 52ae41804..206f03c57 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni CI 规则" created: 2026-08-23 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", "PR #6468", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/runtime.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -84,13 +84,13 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 ## OMNI-CI-2b — 并行测试基础设施隔离任务失败和共享状态 -- 触发：xdist、共享下载 fixture、长寿命 judge/transcriber worker 或进程池 retry。 +- 触发：xdist、共享下载 fixture、长寿命 judge/transcriber worker、进程池 retry，或 spawned `torch.distributed` process group。 - 强制：任务级异常后丢弃污染 worker；submit/result 串行，只有进程崩溃可 retry-once；传给 - xdist 的参数可序列化，依赖固定兼容 major；共享文件在读锁前显式检查存在，写入保持锁语义。 + xdist 的参数可序列化，依赖固定兼容 major；共享文件在读锁前显式检查存在，写入保持锁语义。spawned `torch.distributed` 必须为每个 group 传入新鲜 `file://` rendezvous URL，不能先释放 TCP port 再把其数字交给子进程；`get_open_port` 只用于真实 TCP service。 - 禁止：普通 task error 复用同一进程；为 pytest 生命周期不可达的竞态堆测试；把并行参数或 - 活对象隐式跨进程传递。 + 活对象隐式跨进程传递；把 file rendezvous 套用于 Mooncake/RDMA 等真实监听服务。 - 验收：OOM、普通异常和 BrokenProcessPool 分别验证隔离/重试；online/offline xdist 均通过， - 并发 cache miss 只产生一个完整文件。 ^[PR #6208] ^[PR #6339] + 并发 cache miss 只产生一个完整文件；spawned Gloo/NCCL group 使用独立 file rendezvous，真实 TCP service 仍使用 port helper。 ^[PR #6208] ^[PR #6339] ^[PR #6468] ## OMNI-CI-2c — 昂贵 engine fixture 复用必须恢复状态并隔离拓扑 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0161080a6..07ee6eab5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3ad64c6a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b2c93765` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 015fe21e6..9229d9dbe 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3ad64c6a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b2c93765` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index bb4456b9d..44e836e6e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3ad64c6a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b2c93765` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4870626b8..96dd213ab 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3ad64c6a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b2c93765` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ec43d337f..b5974306e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3ad64c6a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b2c93765` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 644d9b70e..c2bb5f79d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3ad64c6a` 验证存在 +- 源码校验：以上路径均已在 `main @ b2c93765` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 53c3b97a8..5ae844354 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3ad64c6a`（2026-08-22）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b2c93765`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -88,8 +88,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 855719e3 \ - --to 3ad64c6a \ + --from 3ad64c6a \ + --to b2c93765 \ --repo \ --mode report-only ``` From a823285d6178c882849296b9526efb232c0b029d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 11:22:58 +0800 Subject: [PATCH 268/465] knowledge(vllm-omni): sync 2e096788 (PR #6362) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-Commit: 2e096788a6337b0bf7ac22c30bb9afa2afb8f3d4 Upstream-PR: #6362 Knowledge-Delta: 移除已删除的 SoulX-Singer owner、registry/deploy 目录项和陈旧比较引用，并同步 diffusion、pipeline、deploy inventories。 --- adapters/vllm_omni/release_baseline.yaml | 16 ++--- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 25 ++++--- .../vllm-omni/models/gr00t/architecture.md | 2 +- .../vllm-omni/models/soulx-singer/_index.md | 69 ------------------- .../models/soulx-singer/architecture.md | 69 ------------------- 12 files changed, 28 insertions(+), 167 deletions(-) delete mode 100644 knowledge/repos/vllm-omni/models/soulx-singer/_index.md delete mode 100644 knowledge/repos/vllm-omni/models/soulx-singer/architecture.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 667db6e5c..96f008b6e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3ad64c6a1236dfdeb26354f8e4be72a97a4c1a04 + previous_audited_sha: b2c93765393af9d8f8b898db25f958a890ce9ddb audited_ref: main - audited_sha: b2c93765393af9d8f8b898db25f958a890ce9ddb + audited_sha: 2e096788a6337b0bf7ac22c30bb9afa2afb8f3d4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,14 +13,14 @@ inventories: count: 88 sha256: 96d973728f1d67a959121c4aa0d8ba86cff32043ec6f90458f48d1d766acb8af diffusion: - count: 67 - sha256: 9c7160d18bdfff9158738a6ea74a280d52d541a45d9c298892773278594fe68f + count: 65 + sha256: 1ec39b119ad2e9254c0e38c1ae881535811d1ae95c16cebee9daa6d9759f7853 pipelines: - count: 59 - sha256: c2516b9fb6124aa0774a05390a6f05c08ed6998e287b03c8252d85338934933e + count: 57 + sha256: 1cd76b7f5f06207b71e892ec48bc50fbb01a71ee675c6404fc49c7942c8edd52 deploy_yamls: - count: 88 - sha256: d4a1840627fced8bac9b089fe8b13b69bd39ebee5264fd465d42f6c389d094e5 + count: 86 + sha256: 52b7c4bbb29143a224f645afc9047c14d9a8c94d2d709011726236cefec34184 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 83a152fc7..c2e9afd50 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b2c93765393af9d8f8b898db25f958a890ce9ddb`. The canonical machine baseline +2e096788a6337b0bf7ac22c30bb9afa2afb8f3d4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 07ee6eab5..1df69e599 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b2c93765` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2e096788` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9229d9dbe..3182b58bf 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b2c93765` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 2e096788` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 44e836e6e..0a5536d15 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b2c93765` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2e096788` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 96dd213ab..aab1953d5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b2c93765` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2e096788` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b5974306e..130f9cbd6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b2c93765` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2e096788` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c2bb5f79d..553713e04 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b2c93765` 验证存在 +- 源码校验：以上路径均已在 `main @ 2e096788` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5ae844354..8379c7ca0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -1,7 +1,7 @@ --- title: "模型代码入口与 registry 快照" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-04 type: guide tags: [vllm-omni, models] sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/] @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b2c93765`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2e096788`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -36,9 +36,9 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 29 个模型族目录 | -| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 60 条 pipeline / 39 个模型族目录 | -| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 53 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 81 份 | +| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 58 条 pipeline / 38 个模型族目录 | +| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 51 个 key | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 79 份 | 对比上一审计快照（`807db6ef`）：AR 架构 72→77，diffusion pipeline 61→58， OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusion 新增 @@ -55,16 +55,15 @@ mammoth_moda2、mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、moss_tt moss_tts_nano、omnivoice、personaplex、qwen2_5_omni、qwen3_omni、qwen3_tts、step_audio2、 voxcpm2、voxtral_tts -## Diffusion 模型族（39） +## Diffusion 模型族（38） audiox、bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamid_omni、 dreamzero、ernie_image、flux、flux2、flux2_klein、glm_image、gr00t、helios、 hidream_image、hunyuan_image3、hunyuan_video、internvla_a1、krea2、lance、 lingbot_video、longcat_image、longcat_video、ltx2、magi_human、ming_flash_omni、nextstep_1_1、omnigen2、 -omnivoice、ovis_image、qwen_image、sd3、sdxl、sensenova_u1、soulx_singer、 -stable_audio、wan2_2、z_image、minimax_h3 +omnivoice、ovis_image、qwen_image、sd3、sdxl、sensenova_u1、stable_audio、wan2_2、z_image、minimax_h3 -## OMNI_PIPELINES key（53） +## OMNI_PIPELINES key（51） Gr00tN1d7（注意:唯一 CamelCase key）、aura_omni、audex_s2s、audex_thinker_only、 audex_tta、audex_tts、bagel、bagel_single_stage、 @@ -76,20 +75,20 @@ ming_flash_omni_image、ming_flash_omni_thinker_only、ming_flash_omni_tts、 ming_tts、ming_tts_moe、minicpmo_4_5、moss_tts_delay、moss_tts_local、 moss_tts_nano、moss_tts_realtime、omnivoice、personaplex、qwen2_5_omni、 qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_tts、 -soulxsinger_svc、soulxsinger_svs、step_audio_2、step_audio_2_asr、voxcpm2、 +step_audio_2、step_audio_2_asr、voxcpm2、 voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 注意：单 stage diffusion 模型**多数不在** `OMNI_PIPELINES`（引擎为它们生成 默认 diffusion stage 配置,见 [Config 组件](../components/configuration/architecture.md)）; -但存在例外——omnivoice、soulxsinger、Gr00tN1d7、lance、dreamzero 等单 stage +但存在例外——omnivoice、Gr00tN1d7、lance、dreamzero 等单 stage 家族也有显式 key,勿以"在不在 OMNI_PIPELINES"倒推 stage 数。 ## 重派生方法 ```bash python tools/audit_vllm_omni_release.py \ - --from 3ad64c6a \ - --to b2c93765 \ + --from b2c93765 \ + --to 2e096788 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/gr00t/architecture.md b/knowledge/repos/vllm-omni/models/gr00t/architecture.md index 165e677b3..88221f1ec 100644 --- a/knowledge/repos/vllm-omni/models/gr00t/architecture.md +++ b/knowledge/repos/vllm-omni/models/gr00t/architecture.md @@ -22,7 +22,7 @@ sources: [vllm_omni/diffusion/models/gr00t/pipeline_gr00t.py, vllm_omni/diffusio `DiffusionOutput(output={"actions": dict[str, np.float32 ndarray]})` （把动作放进 `output` 以通过引擎空输出守卫——与 DreamZero OpenPI 策略 同形）。 -- **自加载权重（比 SoulX 更严格）**：`weights_sources=()` 且 `load_weights` +- **完全自加载权重**：`weights_sources=()` 且 `load_weights` 喂张量直接 raise——`Gr00tPolicy` 自己 `AutoModel.from_pretrained`（bf16）; 框架 loader 必须置身事外。 - backbone `_Qwen3VLBackbone`：`Qwen3VLForConditionalGeneration` 截到 diff --git a/knowledge/repos/vllm-omni/models/soulx-singer/_index.md b/knowledge/repos/vllm-omni/models/soulx-singer/_index.md deleted file mode 100644 index b21d9a267..000000000 --- a/knowledge/repos/vllm-omni/models/soulx-singer/_index.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: "SoulX-Singer（SVS/SVC 歌声流匹配 + 内联神经预处理）" -created: 2026-07-21 -updated: 2026-07-21 -type: index -tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/soulx_singer/, vllm_omni/model_executor/models/soulx_singer/pipeline.py, vllm_omni/deploy/soulxsinger_svs.yaml] ---- - -# SoulX-Singer - -以下事实在 `main @ 5d44868e` 复核。 - -## 名称与范围 - -- 正式名称 SoulX-Singer（Soul-AILab）歌声家族,两条单 stage diffusion - pipeline（各自单 stage 0 `dit`,`execution_type=DIFFUSION`,final 音频 - 输出,**无 connectors**）：**SVS**（乐谱/歌词驱动合成;registry 架构键 - `SoulXSingerPipeline`,pipeline key `soulxsinger_svs`,pre-process - `get_soulxsinger_pre_process_func`）与 **SVC**（歌声转换;架构键 - `SoulXSingerSVCPipeline`,key `soulxsinger_svc`,pre-process - `get_soulxsinger_svc_pre_process_func`）——这些是架构/注册标识,不是模型 - 别名。 - **命名陷阱**：registry key `SoulXSingerPipeline`（checkpoint config.json - 的 architectures 值）加载的类是 `PipelineSoulXSingerSVS`; - `SoulXSingerSVCPipeline` → `PipelineSoulXSingerSVC`——grep 时勿被 - key/类名错位误导。 -- 虽是单 stage diffusion,**两个 key 都在 `OMNI_PIPELINES`**;少数注册了 - diffusion **pre**-process 的音频家族——神经预处理栈在 `pre_process_func` - 内**懒加载内联运行**,产出音频/F0 载荷,且按变体取用：SVS 走歌词 ASR - （Paraformer/Parakeet）+ ROSVOT MIDI/note 条件;SVC 走 RMVPE F0, - **MIDI 强制关闭**;人声分离（BS-RoFormer）对两者都是可选项 - （`vocal_sep`）。**冻结 Whisper 特征不在预处理里,而由 - `PipelineSoulXSingerSVC` 在条件构建阶段计算**。 -- 入口路径：registry `vllm_omni/diffusion/registry.py` 与 - `vllm_omni/config/pipeline_registry.py`;拓扑 - `model_executor/models/soulx_singer/pipeline.py`;实现 - `diffusion/models/soulx_singer/`（base/svs/svc 三个 pipeline 文件 + - `modules/` + `preprocess/`）;deploy - `vllm_omni/deploy/soulxsinger_{svs,svc}.yaml`;无 stage input processor、 - 无 AR 入口。 -- **双仓权重** + 一个 HF 不发的文件：`Soul-AILab/SoulX-Singer`（合成）+ - `Soul-AILab/SoulX-Singer-Preprocess`（预处理）,另需从 GitHub 手拷 - `phoneme/phone_set.json`（e2e 测试的报错信息就是这么教的——部署陷阱）。 -- 依赖共享模块：[Diffusion 组件](../../components/diffusion/_index.md) - （CFGParallelMixin、组件发现/offload 分组）。 - -## 目录内容 - -| 遇到什么 | 查看哪里 | 说明 | -|---|---|---| -| 三层输入、prompt 拼接 inpainting、自管权重 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | - -## 配置与 checkpoint 差异 - -- 两份 deploy **只差 `pipeline:` 键**（fp16、32 步、guidance 3.0、单卡 - `gpu_memory_utilization: 0.5`、seed 42）,**都不 pin checkpoint ID**; - 离线 README 要求分别建 SVS/SVC 的 "view" 目录并手拷 `phone_set.json`。 -- extra-body 面在 pipeline 类的 ClassVar（无 model_extras 文件）：SVS 收 - metadata/语言/control/max_merge_duration…出 `f0_shift`;SVC 收 wav/F0 路径 - 出 `pitch_shift`（`auto_shift` 支持,SVC 移调 = `pitch_shift×5` 粗 F0 bin）。 -- 自动化测试仅见离线 e2e（`test_soulxsinger_expansion.py`）;本次调查未发现在线示例 - 的 CI 覆盖。 - -## 什么时候查这里 - -- 审查 soulx 的预处理栈、双仓布局或 SVS/SVC 条件差异;新加"带内联预处理的 - diffusion 家族"时以本家族为形态参考。 -- 语义验收见 [model-validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/soulx-singer/architecture.md b/knowledge/repos/vllm-omni/models/soulx-singer/architecture.md deleted file mode 100644 index 8f46d6656..000000000 --- a/knowledge/repos/vllm-omni/models/soulx-singer/architecture.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: "SoulX-Singer 架构" -created: 2026-07-21 -updated: 2026-07-21 -type: architecture -tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_base.py, vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_svs.py, vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_svc.py, vllm_omni/diffusion/models/soulx_singer/utils.py] ---- - -# SoulX-Singer 架构 - -事实在 `main @ 5d44868e` 复核;入口/命名陷阱速览见 [index](_index.md)。 - -## 模型专有部分与共享模块的边界 - -- 共享基类 `FlowMatchingAudioPipeline`：24 kHz,`_DEFAULT_RESCALE_CFG 0.75`; - DiT 估计器是 NAR Llama 变体（`DiffLlama`/`LlamaNARDecoderLayer`）+ CFM - 解码器;Vocos 系声码器。**自管权重**：`weights_sources=()` 完全绕开框架 - diffusers loader,`.pt` 在 `__init__` 里 strict 加载;fp16 部署内 mel+ - vocoder 强制 fp32（`_build_fp32_audio_modules`）。 -- SVS 专有:note 三编码器（音素/音高/类型）+ preflow ConvNeXtV2 + f0 - 编码器;条件经 `mel2note` gather 扩到 mel 帧（溢出 warn+clamp）。 -- SVC 专有:冻结 **Whisper 编码器**特征（prompt+target）+ 粗 F0 嵌入 - （361 bin,`f0_to_coarse`）。 -- 神经预处理栈（`modules/preprocess/`,注意与同级的非神经 `preprocess/` - helper 目录区分）：funasr、NeMo Parakeet（带 CUDA-graph 禁用 - workaround）、BS-RoFormer pip 包这些外部依赖**只被预处理使用**; - `VocalSegmenter` 是规则式(非 NN)。 -- 共享框架面：[Diffusion 组件](../../components/diffusion/_index.md) - （CFGParallelMixin——guidance 可跨卡分片,默认单卡）。 - -## 配置、checkpoint 和兼容范围 - -- kind 判定：checkpoint config.json `architectures` == - `["SoulXSingerPipeline"]`（SVS）或 `["SoulXSingerSVCPipeline"]`（SVC） - （`utils.py::resolve_soulx_kind`）;extra_args 按 kind 白名单校验。 -- 权重根:`resolve_preprocess_weights_root`（双仓布局）; - `phoneme/phone_set.json` 必须手工从 GitHub 拷入。deploy YAML **不含 - checkpoint ID**,离线 README 要求分别建 SVS/SVC 的 "view" 目录。 -- 两个模块各定义同名 `get_soulxsinger_post_process_func`——registry 按各自 - 模块解析,但重名对 `_DIFFUSION_MODELS` 映射改动脆弱（评审注意）。 - -## 从输入到输出的主要流程 - -1. **三层输入分级**（`pre_process_func`）：原始音频（首见时懒建预处理栈, - 按变体内联跑——分离是 `vocal_sep` 可选;SVS 加 ASR+MIDI,SVC 加 RMVPE - F0）产出音频/F0 载荷 / 预计算载荷（`has_precomputed`）/ warmup 哑载荷; - 产物统一挂在 `prompt.additional_information["soulx_preprocessed"]`。 -2. 条件构建:SVS 走乐谱/音素路径;**SVC 在此阶段用冻结 Whisper 提特征** - （强制关 `midi_transcribe`）。 -3. **prompt 拼接 inpainting**：prompt（参考）与 target 沿时间轴在条件空间 - 拼接,CFM 解码器在给定 prompt mel 的条件下生成 target 区段——**无说话人 - 嵌入的音色迁移**。 -4. 声码器出音频;SVC 输出按 target 长度修剪/补齐;post-process 仅 - tensor→numpy。 - -## 怎样验证功能、精度和性能 - -自动化测试仅覆盖离线 e2e,另有一个可复跑基准;本次调查未发现在线链路的 -CI 覆盖——精度/性能结论需另行实测。 - -- e2e：`tests/e2e/offline_inference/test_soulxsinger_expansion.py`（快照双仓、建 - SVS/SVC view 目录、校验 phone_set.json;资产 - `tests/assets/soulxsinger/`）;示例 - `examples/offline_inference/text_to_speech/soulxsinger/end2end.py` 与 - `benchmark.py`（RTF+分段计时）;在线 - `examples/online_serving/text_to_speech/soulxsinger/`（未被 CI 覆盖）。 -- 已知未决：`audio_sample_rate` ClassVar 与 checkpoint `config.yaml` 的 - 优先级未追;在线服务链路无测试佐证。 From b30ed331314250d246d6cd4440b1eb0bf9af6b37 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 11:30:22 +0800 Subject: [PATCH 269/465] knowledge(vllm-omni): sync 0db262de (PR #6311) Upstream-Commit: 0db262de2af7dc4f32002bd57bbb40f44b134613 Upstream-PR: #6311 Knowledge-Delta: document scheduled L1/L4/L5 coverage ownership and repair moved Cosmos3 test sources --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 26 ++++++++++++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/session-state.md | 4 +-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../repos/vllm-omni/models/cosmos3/rules.md | 3 +-- 12 files changed, 40 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 96f008b6e..52741dbbf 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b2c93765393af9d8f8b898db25f958a890ce9ddb + previous_audited_sha: 2e096788a6337b0bf7ac22c30bb9afa2afb8f3d4 audited_ref: main - audited_sha: 2e096788a6337b0bf7ac22c30bb9afa2afb8f3d4 + audited_sha: 0db262de2af7dc4f32002bd57bbb40f44b134613 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c2e9afd50..8be6a1178 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2e096788a6337b0bf7ac22c30bb9afa2afb8f3d4`. The canonical machine baseline +0db262de2af7dc4f32002bd57bbb40f44b134613`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 206f03c57..56c5820a2 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6339", "PR #6343", "PR #6468", .buildkite/cuda/test-nightly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/runtime.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/runtime.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -18,6 +18,7 @@ confidence: high |---|---|---| | 新模型、nightly、CUDA/NPU、硬件 marker、镜像升级 | `OMNI-CI-1a` | `.buildkite/**` → `pyproject.toml` markers → 目标 e2e/accuracy test | | regression/guard、route census、middleware、mutation test | `OMNI-CI-1b` | 公开 app/handler → guard test；先证明旧实现会失败 | +| `NIGHTLY`/`WEEKLY`/`NON_CRITICAL`、L1/L4/L5、coverage、scheduled bootstrap | `OMNI-CI-1c` | `upload_pipeline.py` → bootstrap YAML → ready/merge/nightly/weekly YAML → coverage helper | | pre-commit、SPDX、shellcheck、stability marker | `OMNI-CI-2a` | `.pre-commit-config.yaml`、`.buildkite/**`、`tools/**` | | xdist、共享 worker、下载 fixture、进程池 | `OMNI-CI-2b` | `tests/conftest.py`、`tests/helpers/**`、`tests/model_tests/**` | | 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | @@ -61,6 +62,29 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 证明 route propagation，并用 live assembled app 触发真实 backend condition、核对结构化 status/body 与失败后的 state。 ^[PR #3422] ^[PR #5074] ^[PR #5670] ^[PR #5713] ^[PR #6202] +## OMNI-CI-1c — scheduled lane 与 coverage owner 必须按完整 gate 矩阵验证 + +- 触发：修改 Buildkite bootstrap 的 branch/env/PR-label 条件、L1/L4/L5 workload 归属、 + `--e2e` 上传、coverage helper，或 docs/skip-mark-only 的 scheduled escape hatch。 +- 强制：`main + NIGHTLY=1` 只拥有 L4 nightly；`main + WEEKLY=1` 上传 L5 weekly，且为 + CUDA ready/merge 加 `--e2e` 跑完整 L2/L3 E2E 与 per-mode coverage； + `main + NON_CRITICAL=1` 只上传 weekly 的 non-critical E2E group。普通 main merge 与 + 非 main PR labels 必须继续走各自 gate，NPU scheduled nightly 不隐式上传 ready。 + docs-only/skip-mark-only diff 可以跳过默认 CI，但不能吞掉这些 main scheduled lane。 +- coverage 所有权：CUDA/AMD ready/merge 的 L1 pytest 不附带 `--cov`；weekly CPU job 对 + `tests/ -m 'core_model and cpu'` 生成合并报告。`run_cov_split.sh` 只在 + `WEEKLY=1 && BUILDKITE_BRANCH=main` 拆分 offline/online coverage 与 artifact；其他调用 + 必须保持一次 combined pytest、无 coverage。为 whole-tree collection 移动测试或延迟导入 + example 时，仍须保持原 marker 与测试语义。 +- 禁止：把 `nightly-test` label 当 weekly CPU coverage gate；让 `NIGHTLY=1` 顺带占用 + L2/L3；用 pipeline 已上传推断嵌套 group 一定执行；通过 `pytest --collect-only | grep ...` + 枚举文件却不传播 collection 失败或不检查结果非空。PR #6311 合并时 weekly TTS 循环仍有 + 这个 fail-green 缺口，不能把该 lane 的绿色结果当作完整 TTS collection 证据。 +- 验收：对 docs-only、CI-level-only、普通 main、三种 schedule env、CUDA/NPU 与每个 PR label + 逐项渲染 bootstrap，并断言 child upload、`--e2e` 和 group-level `if`；coverage helper 分别 + 验证 weekly-main 拆分上传与其他环境 combined/no-cov。任何动态 collection 都要让 import + error、零 selection 和单文件失败使 job 红。^[PR #6311] + ## OMNI-CI-2a — CI 工具、schema 和 hook 是可复现供应链 - 触发：修改 pre-commit/Buildkite schema、下载 lint 工具、SPDX/import 检查或 stability marker。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1df69e599..fe4adb4de 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2e096788` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0db262de` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3182b58bf..49187e32d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2e096788` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 0db262de` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/session-state.md b/knowledge/repos/vllm-omni/components/diffusion/session-state.md index 08499d40c..0d80ee8a7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/session-state.md +++ b/knowledge/repos/vllm-omni/components/diffusion/session-state.md @@ -1,10 +1,10 @@ --- title: "World-model session state" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: architecture tags: [vllm-omni, components, diffusion] -sources: ["PR #4657", docs/features/session_state_manager.md, vllm_omni/experimental/world_models/session_state/, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py] +sources: ["PR #4657", docs/features/session_state_manager.md, vllm_omni/experimental/world_models/session_state/, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0a5536d15..79dd8446f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2e096788` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0db262de` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index aab1953d5..1e86f369b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2e096788` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0db262de` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 130f9cbd6..29d2fdb0e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2e096788` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0db262de` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 553713e04..dbe7f5067 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2e096788` 验证存在 +- 源码校验：以上路径均已在 `main @ 0db262de` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8379c7ca0..cc7032a6a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2e096788`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0db262de`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from b2c93765 \ - --to 2e096788 \ + --from 2e096788 \ + --to 0db262de \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index 35ab12f92..4c34d1231 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325"] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325"] confidence: high --- @@ -124,4 +124,3 @@ confidence: high - 强制：Transfer 必须根据控制输入和选定 resolution 从 `VIDEO_RES_SIZE_INFO` 选择 canonical `(height, width, aspect_ratio)` bucket；原始请求尺寸先保存到 `COSMOS3_TRANSFER_REQUESTED_SIZE_KEY`，仅用于冲突告警，不能改变控制输入选出的几何。Transfer JSON prompt、以及 I2V/V2V 生成 canvas 的 aspect ratio 元数据必须使用 canonical ratio；若请求或 JSON prompt 的 ratio 与 canvas 冲突，所有 rank 都重写元数据，只有 rank 0 输出 warning。非 canonical canvas 必须映射到最近的受支持 ratio，而不是使用 gcd 形成新标签。 - 禁止：把 Transfer 的任意 `size=HxW` 当作最终几何或宣称其会被采用；保留控制输入已覆盖的 stale `aspect_ratio`；将元数据重写也放在 `_is_rank_zero()` 条件内导致 rank 间 token ids 不一致；用非 canonical 的精确 gcd ratio 触发无意义的冲突告警。 - 验收：覆盖 Transfer 请求尺寸与控制 bucket 不一致、请求/JSON ratio 冲突、rank 0 与非 rank 0 的 warning/rewrite 行为，以及 `1104x816`、`832x468` 等近似 canvas 映射到 `4,3`、`16,9` 的回归；JSON prompt 解析后必须逐 rank 得到相同 canonical `aspect_ratio`。^[PR #6325] - From d177cecea9253a9d89187416d968d02e7d22f610 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 11:38:29 +0800 Subject: [PATCH 270/465] knowledge(vllm-omni): sync 39c16d75 (PR #6424) Upstream-Commit: 39c16d75a99427e3d8ff04b1d70b6c925475dfc6 Upstream-PR: #6424 Knowledge-Delta: capture CosyVoice3 sampling geometry, typed stage handoff, and causal HiFT device contracts --- adapters/vllm_omni/manifest.yaml | 4 +-- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-runtime-hot-paths.md | 27 +++++++++++++++---- .../components/model-executor/rules.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/cosyvoice3/_index.md | 16 +++++------ .../models/cosyvoice3/architecture.md | 19 +++++++++---- 14 files changed, 58 insertions(+), 34 deletions(-) diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index 3469ae77d..b162524d1 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -72,14 +72,14 @@ modules: wave: 1 risk: high model_executor: - local_paths: [vllm_omni/model_executor/, vllm_omni/experimental/, vllm_omni/model_extras/] + local_paths: [vllm_omni/model_executor/, vllm_omni/experimental/, vllm_omni/model_extras/, vllm_omni/transformers_utils/] upstream_paths: [vllm/model_executor/] test_paths: [tests/model_executor/, tests/diffusion/, tests/e2e/offline_inference/] import_check: "from vllm_omni.model_executor.models.registry import *; print('Model Executor imports OK')" wave: 1 risk: high input_output: - local_paths: [vllm_omni/inputs/, vllm_omni/outputs/, vllm_omni/utils/mm_outputs.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/serialization.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_engine_core_client.py, vllm_omni/engine/output_processor.py, vllm_omni/request.py] + local_paths: [vllm_omni/inputs/, vllm_omni/outputs/, vllm_omni/utils/mm_outputs.py, vllm_omni/data_entry_keys.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/serialization.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_engine_core_client.py, vllm_omni/engine/output_processor.py, vllm_omni/request.py] upstream_paths: [vllm/inputs/, vllm/v1/engine/input_processor.py, vllm/v1/engine/output_processor.py, vllm/v1/request.py] test_paths: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/test_async_omni_engine_input.py, tests/test_outputs.py] import_check: "from vllm_omni.inputs.data import *; from vllm_omni.inputs.preprocess import *; from vllm_omni.engine.async_omni_engine import AsyncOmniEngine; from vllm_omni.engine.orchestrator import Orchestrator; from vllm_omni.engine.serialization import serialize_additional_information; print('Input/Output imports OK')" diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 52741dbbf..31a788055 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2e096788a6337b0bf7ac22c30bb9afa2afb8f3d4 + previous_audited_sha: 0db262de2af7dc4f32002bd57bbb40f44b134613 audited_ref: main - audited_sha: 0db262de2af7dc4f32002bd57bbb40f44b134613 + audited_sha: 39c16d75a99427e3d8ff04b1d70b6c925475dfc6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8be6a1178..8430f9989 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0db262de2af7dc4f32002bd57bbb40f44b134613`. The canonical machine baseline +39c16d75a99427e3d8ff04b1d70b6c925475dfc6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fe4adb4de..8fe05c8ce 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0db262de` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 39c16d75` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 49187e32d..39644b20e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0db262de` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 39c16d75` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 79dd8446f..757aae353 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0db262de` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 39c16d75` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1e86f369b..15a8602ee 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0db262de` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 39c16d75` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md index 07e41005e..8e3b1c196 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md @@ -4,13 +4,13 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048", "PR #6454", "vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py", "PR #6458"] +sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048", "PR #6424", "PR #6454", vllm_omni/data_entry_keys.py, "vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py", "vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py", "vllm_omni/model_executor/stage_input_processors/cosyvoice3.py", "PR #6458"] confidence: high --- # 运行时热路径合同 -`EXEC-8a`、`EXEC-9a`、`EXEC-11a`–`EXEC-11b`：采样循环的不变量提取、固定输入缓存与掩码/精度边界，以及连续 AR 音频侧路和 codec 帧账本。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 +`EXEC-8a`、`EXEC-9a`、`EXEC-11a`–`EXEC-11h`：采样循环的不变量提取、固定输入缓存与掩码/精度边界，以及连续 AR 音频侧路和 codec 帧账本。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 ## EXEC-8a — DiT Euler 采样循环必须只计算一次不变条件 @@ -66,9 +66,9 @@ confidence: high ## EXEC-11f — 音频 STFT 运行时窗口必须随模块迁移 - 触发：模型的声码器或 HiFT 等 STFT 路径在 module `.to(...)`、多设备 stage 或 CUDA 推理中使用由 `__init__` 创建的运行时 tensor，尤其是子类绕过基类初始化时。 -- 强制：将不属于 checkpoint 的运行时 tensor 通过 `register_buffer(..., persistent=False)` 注册到实际消费它的 module，使 device/dtype 迁移覆盖所有初始化路径，并保持其与 STFT 输入同设备。 -- 禁止：把设备敏感 tensor 留作普通 attribute，依赖每次调用临时 `.to(device)` 兜底，或只修基类初始化而遗漏 causal/subclass 路径；不得把派生 window 写入 checkpoint state dict。 -- 验收：构造 CosyVoice3 `CausalHiFTGenerator`，执行 `.to(cuda)` 后断言 `stft_window.device` 与 STFT 输入一致并完成 causal inference；CPU 路径仍可运行，`persistent=False` buffer 不出现在 checkpoint state dict，并覆盖 subclass constructor。 ^[PR #6454] +- 强制：将不属于 checkpoint 的运行时 tensor 通过 `register_buffer(..., persistent=False)` 注册到实际消费它的 module，使常规 `.to(...)` 能迁移它；STFT use site 还必须在 window 与输入设备不同时迁移并把结果保留回 buffer，因为 dummy loader 可能不调用 weight-loading hook。 +- 禁止：把设备敏感 tensor 留作普通 attribute，只在 weight-loading hook 迁移，或每次返回一个不保留的一次性 `.to(device)` 结果；不得只修基类初始化而遗漏 causal/subclass 路径，也不得把派生 window 写入 checkpoint state dict。 +- 验收：构造 CosyVoice3 `CausalHiFTGenerator`，分别覆盖 module `.to(cuda)` 与 dummy-loader 风格的首次 use-site mismatch，断言后续 `_stft/_istft` 复用同设备 buffer 并完成 causal inference；CPU 路径仍可运行，`persistent=False` buffer 不出现在 checkpoint state dict，并覆盖 subclass constructor。 ^[PR #6454] ^[PR #6424] ## EXEC-11g — MiniCPM-o Talker codec penalty 与终止预算必须在 Sampler 前后闭环 @@ -77,3 +77,20 @@ confidence: high - 禁止：把 vLLM whole-stream presence penalty 当作上游 16-frame codec penalty；让 forced EOS 被 `min_tokens` 再次屏蔽后以任意 codec id 继续解码；让无 EOS 的离线请求占满剩余上下文；用本次 simplex 修复改变 native duplex chunk 语义。 - 验收：覆盖正负 logits、空 history、超过 16 frame 的遗忘、`no_penalties`、逐请求 penalty 与单次消费；覆盖 forced/unforced EOS、`min_tokens`、2048 上限和剩余 context clamp，并用真实长文本 TTS 检查请求在预算边界释放且无静默尾部。^[PR #6458] +## EXEC-11h — CosyVoice3 sampler 与 stage handoff 必须保留有限 logits 和 typed nested payload + +- 触发：修改 CosyVoice3 talker sampling、RAS fallback、outer transformer config，或跨 stage 的 + `additional_information` serializer/decoder。 +- 强制：在 request mask 与所有 logits processor 后只让 finite logits 参与 sampling；整行无 + finite candidate 必须显式失败，RAS 为唯一合法 token 恢复时保存 clone 的 score。outer config 的 + attention/KV-head metadata 必须与实际 nested Qwen checkpoint 一致。tensor payload 用 raw bytes、 + recorded PyTorch dtype 与 shape round-trip，并经 shared decoder 重建 nested dotted keys，再由 + stage processor 消费。 +- 禁止：把 NaN/±inf 变为可采样的均匀分布、从 all-`-inf` softmax 继续抽样、让 outer config + 静默退回 query-head 数作为 KV-head 数，或以 NumPy/flat dotted-key local decoder 代替 shared typed + serialization。 +- 验收：覆盖 NaN/±inf、全非有限行、唯一合法 token 的 RAS fallback 和 14 query/2 KV config；BF16 + payload 必须 bit-exact 并在 async handoff 中保留 `embed` conditioning；固定 seed 的 token 序列 + 不得被拿来与旧 `torch.multinomial` 比较。当前实现把 `+inf` 也排除，并在整行无 finite candidate + 时从 `sample()` 抛错、可能终止整个 EngineCore；这两项是 review 接受的 follow-up 风险，不能描述成 + per-request 隔离或通用正无穷采样语义。^[PR #6424] diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index b4c12a6fb..1223e6c2b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -38,7 +38,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `image-task-envelope` | shared image task example 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6b`，见 [image task envelope 合同](rules-image-task-envelope.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | | `output-contract` | Omni 输出类型与字段/复制合同 | `EXEC-7a`–`EXEC-7b`，见 [输出类型合同](rules-output-contract.md) | -| `runtime-hot-paths` | 采样循环不变量、固定输入缓存、AR 音频侧路与 codec 帧账本 | `EXEC-8a`, `EXEC-9a`, `EXEC-11a`–`EXEC-11b`，见 [运行时热路径合同](rules-runtime-hot-paths.md) | +| `runtime-hot-paths` | 采样循环不变量、固定输入缓存、AR 音频侧路、codec 帧账本与 CosyVoice3 typed handoff | `EXEC-8a`, `EXEC-9a`, `EXEC-11a`–`EXEC-11h`，见 [运行时热路径合同](rules-runtime-hot-paths.md) | | `platform-backends` | NPU runner 接口、ROCm 分页注意力、NPU 模型补丁注册 | `EXEC-10a`, `EXEC-12a`–`EXEC-13a`，见 [平台后端合同](rules-platform-backends.md) | ## 严格配置校验 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 29d2fdb0e..66bed6f48 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0db262de` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 39c16d75` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index dbe7f5067..9730d5e00 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0db262de` 验证存在 +- 源码校验：以上路径均已在 `main @ 39c16d75` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index cc7032a6a..131bd9213 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0db262de`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 39c16d75`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 2e096788 \ - --to 0db262de \ + --from 0db262de \ + --to 39c16d75 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md b/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md index 3db3a3d52..e9a40c560 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md @@ -1,15 +1,15 @@ --- title: "CosyVoice3（单架构双 stage,RAS 合并停止,TRT 加速）" created: 2026-07-21 -updated: 2026-09-02 +updated: 2026-09-04 type: index tags: [vllm-omni, models] -sources: ["PR #5869", vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoice3.yaml, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py] +sources: ["PR #5869", "PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoice3.yaml, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py] --- # CosyVoice3 -以下事实在 `main @ fc8946fc` 复核。 +以下事实在 `main @ 39c16d75` 复核。 ## 名称与范围 @@ -41,12 +41,10 @@ sources: ["PR #5869", vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/dep - 树内 `code2wav_core/hifigan.py::HiFTGenerator` 也是 Step-Audio2 与 MiniCPM-o graph wrapper 的 shared consumer：pre-iSTFT magnitude/phase 与 eager iSTFT/clamp 必须保持同一 inference 合同， 修改时三家一起做 state-dict、音频与 graph/eager parity。^[PR #5869] -- 目标 pin 有未闭合的 causal consumer：base `_stft/_istft` 已移除每次 - `stft_window.to(device)`，但 `CausalHiFTGenerator.__init__` 绕过 base initializer，并把 window - 作为普通 tensor 而非 registered buffer；module `.to(cuda)` 不会迁移它，继承的 STFT 路径可能 - CPU/CUDA device mismatch。新增 Step-Audio2 parity 与 MiniCPM graph 测试只实例化 noncausal base； - 修复必须让 causal window 注册/迁移，并增加 CosyVoice3 CUDA causal inference 回归，同时复核 - base 的 MiniCPM、Step-Audio2 与 GLM-TTS consumers。^[PR #5869] +- `CausalHiFTGenerator` 的 STFT window 是 nonpersistent buffer；`_stft/_istft` 在输入设备与 + buffer 不同时迁移并复用它，因此 dummy loader 未调用 weight-loading hook 时也不会保留 CPU window。 + 该 buffer 不进入 checkpoint；CosyVoice3 causal CPU/CUDA 回归应覆盖 module move 与首次 use-site move。 + ^[PR #6424] ## 目录内容 diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md b/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md index b374ff152..a7a386cc3 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md @@ -1,15 +1,15 @@ --- title: "CosyVoice3 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-04 type: architecture tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py, vllm_omni/model_executor/models/cosyvoice3/pipeline.py, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py, vllm_omni/deploy/cosyvoice3.yaml] +sources: ["PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/cosyvoice3/pipeline.py, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py, vllm_omni/deploy/cosyvoice3.yaml] --- # CosyVoice3 架构 -事实在 `main @ 5d44868e` 复核;入口/模式速览见 [index](_index.md)。 +事实在 `main @ 39c16d75` 复核;入口/模式速览见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -32,7 +32,7 @@ sources: [vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py, vllm_omni/mo `FunAudioLLM/Fun-CosyVoice3-0.5B-2512`。 - config（`transformers_utils/configs/cosyvoice3.py`）：`eos_token_id` 默认 6562;注释明确官方把 **所有 ≥ speech_token_size 的 token（6561–6760）都视 - 为停止信号**;`sample_rate 24000`、hidden 896、heads 14 硬编码。 + 为停止信号**;`sample_rate 24000`、hidden 896、query heads 14、KV heads 2 硬编码。 - **RAS 停止机制（停止失效 bug 类别）**：停止 token id **6562 接收 200 个 停止 logit 的 logsumexp 合并分数**;YAML 的 `repetition_penalty: 1.0001` 唯一存在理由是逼 vLLM 维护 @@ -44,7 +44,9 @@ sources: [vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py, vllm_omni/mo ## 从输入到输出的主要流程 1. 文本 + 参考音频作为多模态条件进入（campplus 说话人嵌入与 - speech_feat/speech_token 条件张量打进 `additional_information`）。 + speech_feat/speech_token 条件张量打进 `additional_information`）；共享 serializer 以原始 + `uint8` storage + recorded PyTorch dtype/shape 往返，因此 BF16 保持 bit-exact，shared decoder + 还原 dotted nested keys（如 `embed.speech_token`）后交给 stage processor。 2. talker AR 出 speech token（词表 6561,合并停止 6562）。 3. 交接双注册（**唯一同时注册两条交接的家族**,由 `deploy.async_chunk` 选）: - 流式 `talker2code2wav_async_chunk`：prompt 条件按 chunk 对齐 @@ -68,6 +70,13 @@ gate（YAML 头注的 H100/H20-3e 连续性数字是部署调参记录,非精度 `tests/model_executor/stage_input_processors/test_cosyvoice3_stage_input_processors.py` （资产 `tests/assets/cosyvoice3/zero_shot_prompt.wav`）;示例 `examples/{offline_inference,online_serving}/text_to_speech/cosyvoice3/`。 +- 采样在 request mask 和 logits processor 之后排除 NaN/±inf；整行没有 finite candidate 必须 + fail loudly 而不能抽样随机 speech。该 guard 每 sampler batch 有一次 device sync，且不同于 + `torch.multinomial` 的 vLLM sampler 可改变固定 seed 的 token 序列。RAS 仅剩合法 token 时必须 + 恢复 clone 的原 score，不能从 all-`-inf` softmax 抽样。^[PR #6424] +- 最新 head 的真实权重证据覆盖 H20 CUDA graph/attention 与 MI300X Triton/AITER，dummy 权重也在 + H20/MI300X 通过；没有 H100/MI325 复测，且这些运行都未使用 TensorRT campplus，不能外推到该硬件或 + TRT 路径。`+inf` 被当作非法值排除、无 finite 行会杀死 EngineCore 而非只 abort 单请求，仍是 follow-up。 - 已知未决：6563–6760 的实际停止行为依赖 talker `compute_logits` 的合并 逻辑（只略读）;NPU 侧 `cosyvoice2_dit_attn.py`（名字是 cosyvoice2）是否 接到 v3 DiT 路径未追;code2wav 内 TRT/PyTorch 选择的具体代码未逐行读。 From dacec5d15f354893acf2d42fd9db0b2fb371e8e2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 15:44:06 +0800 Subject: [PATCH 271/465] knowledge(vllm-omni): sync 1e2a6446 (PR #6353) Upstream-Commit: 1e2a6446f79da72b66e15f684d0256244c725916 Upstream-PR: #6353 Knowledge-Delta: remove retired AudioX and MammothModa2 owners, refresh registry inventories, and preserve current Stable Audio/cache contracts --- adapters/vllm_omni/release_baseline.yaml | 20 ++--- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/cache-acceleration.md | 5 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../rules-image-task-envelope.md | 5 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/models/audiox/_index.md | 39 --------- knowledge/repos/vllm-omni/models/catalog.md | 78 +++++++++--------- .../vllm-omni/models/mammoth-moda2/_index.md | 60 -------------- .../models/mammoth-moda2/architecture.md | 82 ------------------- 14 files changed, 61 insertions(+), 242 deletions(-) delete mode 100644 knowledge/repos/vllm-omni/models/audiox/_index.md delete mode 100644 knowledge/repos/vllm-omni/models/mammoth-moda2/_index.md delete mode 100644 knowledge/repos/vllm-omni/models/mammoth-moda2/architecture.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 31a788055..a9e1c0a90 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0db262de2af7dc4f32002bd57bbb40f44b134613 + previous_audited_sha: 39c16d75a99427e3d8ff04b1d70b6c925475dfc6 audited_ref: main - audited_sha: 39c16d75a99427e3d8ff04b1d70b6c925475dfc6 + audited_sha: 1e2a6446f79da72b66e15f684d0256244c725916 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 88 - sha256: 96d973728f1d67a959121c4aa0d8ba86cff32043ec6f90458f48d1d766acb8af + count: 81 + sha256: 9d5a685eae7ef099bb884557ca51b0e1b5f19c8555aa16bbd136c2081d95cd78 diffusion: - count: 65 - sha256: 1ec39b119ad2e9254c0e38c1ae881535811d1ae95c16cebee9daa6d9759f7853 + count: 64 + sha256: 6bebd607c9e9570f6c4622d220a9a7db6f5d07e3d008b1adb6ee526f68af49b8 pipelines: - count: 57 - sha256: 1cd76b7f5f06207b71e892ec48bc50fbb01a71ee675c6404fc49c7942c8edd52 + count: 55 + sha256: 9d5033c9a7ffd7ff636e32db0f9be2a549b7c16b034e13817e280f19a9e16769 deploy_yamls: - count: 86 - sha256: 52b7c4bbb29143a224f645afc9047c14d9a8c94d2d709011726236cefec34184 + count: 84 + sha256: 002a24e33c1369f7e1b0a6824d0d2805c6ffb4791c652d6d67fd161a075017bd # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8430f9989..c7d464410 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -39c16d75a99427e3d8ff04b1d70b6c925475dfc6`. The canonical machine baseline +1e2a6446f79da72b66e15f684d0256244c725916`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8fe05c8ce..07aee9e17 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 39c16d75` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1e2a6446` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 39644b20e..3452cd024 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 39c16d75` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1e2a6446` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md b/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md index 2611dc14e..e421b0829 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md +++ b/knowledge/repos/vllm-omni/components/diffusion/cache-acceleration.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: guide tags: [vllm-omni, components, diffusion] -sources: ["PR #5840", docs/design/feature/cache_dit.md, docs/design/feature/teacache.md, recipes/MiniMaxAI/MiniMax-H3.md, vllm_omni/diffusion/cache/, vllm_omni/diffusion/data.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, tests/diffusion/cache/test_teacache_extractors.py] +sources: ["PR #5840", "PR #6353", docs/design/feature/cache_dit.md, docs/design/feature/teacache.md, recipes/MiniMaxAI/MiniMax-H3.md, vllm_omni/diffusion/cache/, vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, tests/diffusion/cache/test_teacache_extractors.py] --- # 缓存加速（cache_dit / TeaCache / stepcache / magcache） @@ -29,7 +29,8 @@ sources: ["PR #5840", docs/design/feature/cache_dit.md, docs/design/feature/teac `prompt_embed_cache.py`（prompt 嵌入缓存）、`prefix_caching.md`（diffusion 侧 前缀缓存语义）。 - 兼容性例外：registry 的 `_NO_CACHE_ACCELERATION = {"NextStep11Pipeline", - "AudioXPipeline"}`——这两条 pipeline 不支持 cache_dit/tea_cache。 + "Pi0Pipeline", "LingBotWorldCausalDMDPipeline"}`——这三条 pipeline 不支持 + cache_dit/tea_cache。 - MiniMax-H3 只校准 FL2VA：Ref2VA-only fail fast，combined 只 hook `transformer` 并告警 `transformers_ref` 不缓存；具体阈值与证据见 [MMH3-2g](../../models/minimax-h3/rules-cache-task.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state)。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 757aae353..0f4c14c95 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 39c16d75` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1e2a6446` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 15a8602ee..7b0a588c2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 39c16d75` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1e2a6446` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md index c1c39f3ec..46c785ff5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #6049", "PR #6076", vllm_omni/model_extras/, "PR #6070", "vllm_omni/model_extras/registry.py", "PR #5614"] +sources: ["PR #6049", "PR #6076", vllm_omni/model_extras/, "PR #6070", "vllm_omni/model_extras/registry.py", "PR #5614", "PR #6353"] confidence: high --- @@ -22,7 +22,7 @@ confidence: high - 禁止：pipeline 已负责 validation/normalization 时复制 generic builder；让 registry 从零重建 task modality；用 truthiness 丢空 negative prompt；为每个模型复制 Python example。 - 验收：canonical builder 覆盖 omitted/empty/value negative prompt 与 PIL media identity；registry - 覆盖 Bagel、MammothModa2、Ming、VACE custom path及 unknown identity path；Cosmos3/LingBot + 覆盖 Bagel、Ming、VACE custom path及 unknown identity path；Cosmos3/LingBot pipeline tests继续拥有模型专属 validation。^[PR #6049] ## EXEC-6b — model_extras 必须声明模型专有参数与输出张量范围 @@ -40,4 +40,3 @@ confidence: high - 强制：按解析后的 `model_class_name` 在 registry 声明 resolver；公开 accessor 必须把真实 `model` 与 `revision` 传给 model-owned hook；共享 config/serving consumer 只调用 accessor，unknown model 保持安全默认值。 - 禁止：在共享 serving/data 代码中写 LTX class/version 分支；按 repository basename 猜版本；用 class 名代替 checkpoint metadata；丢弃 pinned revision 或把共享 `LTX2Pipeline` 的策略无条件应用于 2、2.3、2.5。 - 验收：覆盖 arbitrary local path、2/2.3/2.5 metadata、Full/distilled pipeline class、unknown class 和 pinned revision；断言 extra-body 声明、transformer subfolder 与 reference-image policy 都经 registry accessor 到达真实 consumer。^[PR #6070] - diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 66bed6f48..d4f6c6b53 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 39c16d75` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1e2a6446` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9730d5e00..def571d9b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 39c16d75` 验证存在 +- 源码校验：以上路径均已在 `main @ 1e2a6446` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/audiox/_index.md b/knowledge/repos/vllm-omni/models/audiox/_index.md deleted file mode 100644 index 04df6e7e7..000000000 --- a/knowledge/repos/vllm-omni/models/audiox/_index.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: "AudioX（文本/视频条件音频 DiT）" -created: 2026-07-21 -updated: 2026-07-21 -type: index -tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/audiox/, vllm_omni/diffusion/registry.py] ---- - -# AudioX - -以下事实在 `main @ 5d44868e` 复核（源码派生页,尚无本模型的运行经验沉淀）。 - -## 名称与范围 - -- 正式名称 AudioX,无别名、无变体有据。diffusion registry: - `AudioXPipeline` →（`audiox`, `pipeline_audiox`, `AudioXPipeline`）,post - `get_audiox_post_process_func`;在 `_NO_CACHE_ACCELERATION` 名单 - （不吃 cache_dit/tea_cache 加速）。 -- 单 stage diffusion,不在 `OMNI_PIPELINES`（引擎默认 diffusion stage 配置, - 见 [Configuration](../../components/configuration/architecture.md)）。无 deploy YAML。 -- 源码：`vllm_omni/diffusion/models/audiox/`（`pipeline_audiox.py` 40 KB + - `audiox_transformer.py`,`MMDiffusionTransformer`）。 -- 依赖共享模块：[Diffusion 组件](../../components/diffusion/_index.md)。 - -## 结构与要点 - -- 条件侧：T5 文本 + `CLIPVisionModelWithProjection` 视频条件,MAF 融合块 - （`_MAFCrossAttentionBlock`/`MAF_Block`）;任务模式 VIDEO_ONLY/TEXT_VIDEO - 来自 `vllm_omni/transformers_utils/processors/audiox.py`。 -- `AutoencoderOobleck` 音频 VAE;采样用 `torchsde` - `_BrownianTreeNoiseSampler` 自定义 SDE——代码注释明确与 diffusers - EDMDPMSolverMultistep 的 v-prediction 预处理有差异,改采样器前先读该注释。 -- `SupportAudioOutput` 输出。 - -## 什么时候查这里 - -- 审查 AudioX 的 SDE 采样、视频条件或 Oobleck VAE 改动;RNG/graph 共享规则见 - [Diffusion rules](../../components/diffusion/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 131bd9213..563265822 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 39c16d75`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1e2a6446`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -35,48 +35,48 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| -| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 29 个模型族目录 | -| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 58 条 pipeline / 38 个模型族目录 | -| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 51 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 79 份 | +| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | +| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 64 条 pipeline / 41 个模型族目录 | +| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 55 个 key | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 84 份 | -对比上一审计快照（`807db6ef`）：AR 架构 72→77，diffusion pipeline 61→58， -OMNI_PIPELINES 46→51，deploy 71→79。新增 AR 家族是 `audex`；diffusion 新增 -`minimax_h3`，同时 LTX-2/LTX-2.3 的五个旧 registry names 合并为两个入口。新增 -pipeline keys 是四个 Audex 模式和 `nemotron_labs_audex` alias；deploy 新增八个 -Audex files 与 `qwen3_omni_moe_thinking.yaml`，删除 `minicpmo_4_5_batching.yaml`。 -本次相对 `72ee535f` 本轮新增两条可复用合同：diffusion 单 GPU 的进程内 executor 生命周期，以及 omitted `distributed_executor_backend` 必须保留到运行时按设备数解析。文档、测试改动和 serving 中仅承担该配置透传的实现细节未单独沉淀为规则，也不能据此推断所有硬件上的性能或生产级支持。 +对比上一审计快照（`39c16d75`）：AR 架构 88→81，diffusion pipeline 65→64， +OMNI_PIPELINES 57→55，deploy 86→84。PR #6353 删除 AudioX 的 diffusion registry、 +processor/extras/recipe/examples/tests，并删除 MammothModa2 Preview/Dev 共用的七个 AR +architecture、两个 pipeline key、两个 deploy YAML 及其 tokenizer/config/AR/DiT/bridge。 +因此 active owner 与 catalog 不再声明这两个家族；diffusion text-to-audio 的 first-party +示例只剩 Stable Audio，Audex 的 TTA 仍是独立两阶段 autoregressive 路径。^[PR #6353] -## AR/omni 模型族（29） +## AR/omni 模型族（31） -aura_omni、audex、bagel、cosyvoice3、covo_audio、dynin_omni、fish_speech、glm_image、 -glm_tts、higgs_audio_v2、higgs_audio_v3、hunyuan_image3、indextts2、indextts2_5、 -mammoth_moda2、mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、moss_tts、 -moss_tts_nano、omnivoice、personaplex、qwen2_5_omni、qwen3_omni、qwen3_tts、step_audio2、 -voxcpm2、voxtral_tts +aura_omni、audex、bagel、cosyvoice3、covo_audio、dots_tts、dynin_omni、fish_speech、 +gepard、glm_image、glm_tts、higgs_audio_v2、higgs_audio_v3、hunyuan_image3、indextts2、 +mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、minimax_music3、moss_tts、 +moss_tts_nano、nemotron_voicechat、omnivoice、personaplex、qwen2_5_omni、qwen3_omni、 +qwen3_tts、step_audio2、voxcpm2、voxtral_tts -## Diffusion 模型族（38） +## Diffusion 模型族（41） -audiox、bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamid_omni、 +bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamid_omni、 dreamzero、ernie_image、flux、flux2、flux2_klein、glm_image、gr00t、helios、 -hidream_image、hunyuan_image3、hunyuan_video、internvla_a1、krea2、lance、 -lingbot_video、longcat_image、longcat_video、ltx2、magi_human、ming_flash_omni、nextstep_1_1、omnigen2、 -omnivoice、ovis_image、qwen_image、sd3、sdxl、sensenova_u1、stable_audio、wan2_2、z_image、minimax_h3 - -## OMNI_PIPELINES key（51） - -Gr00tN1d7（注意:唯一 CamelCase key）、aura_omni、audex_s2s、audex_thinker_only、 -audex_tta、audex_tts、bagel、bagel_single_stage、 -bagel_think、cosyvoice3、covo_audio、dreamzero、dynin_omni、fish_qwen3_omni、 -glm_image、glm_tts、higgs_audio_v2、higgs_multimodal_qwen3、hunyuan_image3_ar、 -hunyuan_image3_dit、hunyuan_image_3_moe、hunyuan_video_15、indextts2、lance、 -mammoth_moda2、mammoth_moda2_ar、mimo_audio、ming_flash_omni、 -ming_flash_omni_image、ming_flash_omni_thinker_only、ming_flash_omni_tts、 -ming_tts、ming_tts_moe、minicpmo_4_5、moss_tts_delay、moss_tts_local、 -moss_tts_nano、moss_tts_realtime、omnivoice、personaplex、qwen2_5_omni、 -qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_tts、 -step_audio_2、step_audio_2_asr、voxcpm2、 -voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 +hidream_image、hidream_o1_image、hunyuan_image3、hunyuan_video、internvla_a1、krea2、lance、 +lingbot_video、lingbot_world、longcat_image、longcat_video、ltx2、magi_human、ming_flash_omni、 +minimax_h3、nextstep_1_1、omnigen2、omnivoice、ovis_image、pi0、qwen_image、sana_wm、sd3、 +sdxl、sensenova_u1、stable_audio、wan2_2、z_image + +## OMNI_PIPELINES key（55） + +Gr00tN1d7（注意:唯一 CamelCase key）、audex_s2s、audex_thinker_only、audex_tta、 +audex_tts、aura_omni、bagel、bagel_single_stage、bagel_think、cosyvoice3、covo_audio、 +dots_tts、dreamzero、dynin_omni、fish_qwen3_omni、gepard、glm_image、glm_tts、 +higgs_audio_v2、higgs_multimodal_qwen3、hunyuan_image3_ar、hunyuan_image3_dit、 +hunyuan_image_3_moe、hunyuan_video_15、indextts2、indextts2_5、lance、mimo_audio、 +ming_flash_omni、ming_flash_omni_image、ming_flash_omni_thinker_only、 +ming_flash_omni_tts、ming_tts、ming_tts_moe、minicpmo_4_5、minimax_music3、 +moss_tts_delay、moss_tts_local、moss_tts_nano、moss_tts_realtime、nemotron_labs_audex、 +nemotron_labs_voicechat、nemotron_voicechat、omnivoice、personaplex、pi0、qwen2_5_omni、 +qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_tts、step_audio_2、 +step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v 注意：单 stage diffusion 模型**多数不在** `OMNI_PIPELINES`（引擎为它们生成 默认 diffusion stage 配置,见 [Config 组件](../components/configuration/architecture.md)）; @@ -87,8 +87,8 @@ voxtral_tts、wan2_2_ti2v、nemotron_labs_audex、indextts2_5 ```bash python tools/audit_vllm_omni_release.py \ - --from 0db262de \ - --to 39c16d75 \ + --from 39c16d75 \ + --to 1e2a6446 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/mammoth-moda2/_index.md b/knowledge/repos/vllm-omni/models/mammoth-moda2/_index.md deleted file mode 100644 index 5f3496f08..000000000 --- a/knowledge/repos/vllm-omni/models/mammoth-moda2/_index.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: "MammothModa2（AR→DiT,但 DiT 是 LLM_GENERATION stage）" -created: 2026-07-21 -updated: 2026-07-21 -type: index -tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/mammoth_moda2/, vllm_omni/diffusion/models/mammoth_moda2/, vllm_omni/deploy/mammoth_moda2.yaml] ---- - -# MammothModa2 - -以下事实在 `main @ 5d44868e` 复核。 - -## 名称与范围 - -- 正式名称 MammothModa2-Preview（ByteDance Research;recipe - `bytedance-research/MammothModa2-Preview`）。 -- AR registry 五个入口（`mammoth_moda2` 家族;**无 diffusion registry 入口**）: - MoE 文本基座 `MammothModa2Qwen2ForCausalLM`、AR 多模态 - `MammothModa2ARForConditionalGeneration`、DiT `MammothModa2DiTPipeline` - （经 15 行 shim 转到 `diffusion/models/mammoth_moda2/`）、路由包装 - `MammothModa2ForConditionalGeneration`（按 `model_stage` 分发 ar/dit; - `"vae"` 保留并 raise）+ HF 别名 `Mammothmoda2Model`。 -- pipeline key 两个：`mammoth_moda2`——stage 0 AR（LLM_AR, - `engine_output_type="latent"`）→ stage 1 DiT（LLM_GENERATION, - `input_sources=(0,)`,经 `stage_input_processors/mammoth_moda2.py::ar2dit` - 交接,图像出）;`mammoth_moda2_ar`——仅 stage 0,文本出。**两 key 同架构 - 同权重**,但拓扑不同：AR-only 把流水线截断到 stage 0、final 输出改为 - text、无 latent 导出要求;deploy 侧参数也不同（AR-only - `max_num_seqs 16` vs 全拓扑 stage0 100/stage1 1）。 -- checkpoint：`bytedance-research/MammothModa2-Preview`（recipe/示例记载, - YAML 不 pin）。 -- 依赖共享模块：AR 侧继承 vLLM Qwen2.5-VL 栈（processor/模型基类,加 - MammothU tokenizer）;AR runner 的 latent 导出; - `stage_input_processors/mammoth_moda2.py`; - [Config 组件](../../components/configuration/architecture.md)。t2i 走共享示例 - `examples/offline_inference/text_to_image/`,AR-only 示例在 - `examples/offline_inference/mammothmodal2_preview/`。 - -## 目录内容 - -| 遇到什么 | 查看哪里 | 说明 | -|---|---|---| -| 双词表/网格约束解码/latent 交接 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | - -## 配置与 checkpoint 差异 - -- **stage 1 不是 diffusion-engine stage**：`LLM_GENERATION` 里跑 - `VllmConfig` 版 DiT——cache-dit/SP/OmniDiffusionRequest 都不适用; - `num_inference_steps` 必须走 extra_body（`--num-inference-steps` 旗标到不了 - DiT,`model_extras/mammothmodal2_preview.py` 注释明示）。 -- `mammoth_moda2.yaml`：两 stage 同卡（AR 0.5 + DiT 0.3 显存配比,均 eager）; - DiT 默认 `text_guidance_scale 9.0`/`cfg_range [0,1]`/`steps 50`; - **无负向 prompt 路径**。 - -## 什么时候查这里 - -- 审查 mammoth_moda2 的双词表、约束解码或 AR↔DiT 交接;评审"diffusion 引擎 - 统一改造"时本家族是必须点名的例外。 -- 语义验收见 [model-validation](../../review/guides/model-validation.md)。 diff --git a/knowledge/repos/vllm-omni/models/mammoth-moda2/architecture.md b/knowledge/repos/vllm-omni/models/mammoth-moda2/architecture.md deleted file mode 100644 index 5de84473a..000000000 --- a/knowledge/repos/vllm-omni/models/mammoth-moda2/architecture.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -title: "MammothModa2 架构" -created: 2026-07-21 -updated: 2026-07-21 -type: architecture -tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py, vllm_omni/diffusion/models/mammoth_moda2/pipeline_mammothmoda2_dit.py, vllm_omni/model_executor/stage_input_processors/mammoth_moda2.py] ---- - -# MammothModa2 架构 - -事实在 `main @ 5d44868e` 复核;行号随源码漂移,改代码前以当前版本为准。 -入口/变体速览见 [MammothModa2 index](_index.md);语义验收方法见 -[model-validation](../../review/guides/model-validation.md)。 - -## 模型专有部分与共享模块的边界 - -- 专有 AR：`mammoth_moda2.py`——**双词表 LM**（`extra_gen_vocab`:独立 - `gen_embed_tokens`/`gen_head`,base+gen logits 拼接以维持 vLLM "logits 维 - == 词表"不变量）;token 级双专家路由 `moe_forward` - （生成 token 走 gen expert,层范围由 `"ffn_attention-14:28"` 风格串门控—— - 是**确定性模态路由,不是学习型 router**）;`gen_token_mask` 随 - `IntermediateTensors` 过 PP。 -- 继承的共享栈：AR 多模态类 subclass vLLM 的 Qwen2.5-VL - （`Qwen2_5_VLForConditionalGeneration` + 对应 processor 族,换上 - MammothU tokenizer 与 `MammothModa2Qwen2ForCausalLM` 语言基座）;跨 stage - 交接责任在 AR runner（latent 导出）+ - `stage_input_processors/mammoth_moda2.py`。 -- 专有 DiT：`diffusion/models/mammoth_moda2/`——Lumina2 系 DiT + - **可选** Q-Former 条件精炼器（仅当 checkpoint 配置有 - `gen_image_condition_refiner_config` 时构建）+ 本地 - `FlowMatchEulerDiscreteScheduler` 移植 + 实值 RoPE（`rope_real.py`）。 - **RoPE 频表刻意取顶层 `config.gen_axes_lens` 而非 checkpoint 的 - `gen_dit_config.axes_lens`**（后者可小到 1024,不够 vLLM - warmup/cudagraph 用——不要"简化"回去）。 -- 权重卫生：AR stage 的 `hf_to_vllm_mapper` 丢弃 **`gen_transformer.` / - `gen_vae.` / `gen_image_condition_refiner.`** 前缀（生成侧的双词表 - 嵌入/头当然保留——那是 AR 自己的）;DiT stage 丢 `llm_model.*`;包装类以 - `ar.`/`dit.` 重前缀——各 stage 只加载共享 checkpoint 的自己那半。 - -## 配置、checkpoint 和兼容范围 - -- config 类 `Mammothmoda2Config`（`transformers_utils/configs/mammoth_moda2.py`）: - 嵌套 `llm_config` + 字典式 `gen_vae_config`/`gen_dit_config`（diffusers - 风格 dict,DiT 组件用 `from_config` 构建）。checkpoint - `bytedance-research/MammothModa2-Preview`;两个 pipeline 变体同权重、 - 不同拓扑（见 index）。 -- `gen_vocab_start_index` 是**单一枢轴常量**：AR 掩码、logits 约束、DiT 条件 - 拆分三处共用,改动必须三处同步。 -- t2i prompt 由 `model_extras/mammothmodal2_preview.py` 组装 - （`<|image start|>{ar_width}*{ar_height}<|image token|>`,`_PATCH_SIZE=16`）; - extra-body 面 `{text_guidance_scale, cfg_range, num_inference_steps}`。 - -## 从输入到输出的主要流程 - -1. AR 前传:t2i 请求做**网格约束解码**——每 `ar_width+1` 个 token 强制 EOL, - 行内经 logits mask 限制到视觉 token 区间;非 t2i 请求整个 gen 词表置 - −inf。逐步由 `runtime_additional_information` 驱动。 -2. `ar2dit`（`stage_input_processors/mammoth_moda2.py`）： - `full_token_ids = prompt + generated[:-1]`（末 token 无 hidden state,丢弃, - 有长度断言守护）;**要求 `completion_output.multimodal_output["latent"]` - 为全量 hidden states,缺失即 raise**——这份 latent 由 **AR runner**（不是 - AR 模型)在 stage 0 `engine_output_type="latent"` 下挂上;**以 float32 - 连续张量跨进程**（numpy 序列化器无 bf16,DiT 侧再转）;文本/图像条件拆分 - 推迟到 DiT 的 `_split_ar_conditions`（从 config token id 重建掩码）。 -3. DiT（LLM_GENERATION runner 内）：caption embedder 重初始化为 - `RMSNorm+Linear(llm→dit)`;CFG 用 `text_guidance_scale`+`cfg_range` 窗口。 - pipeline 构造了 VAE、stage 1 声明 final 图像输出;去噪/VAE 解码/图像打包 - 的逐步实现未逐行核对（见文末未决）。 - -## 怎样验证功能、精度和性能 - -pin 上只有**功能/config 面**的验证入口,没有专门的精度基线或性能 gate 证据; -精度/性能结论需另行实测。 - -- e2e：`tests/e2e/offline_inference/test_mammoth_moda2_expansion.py` - （t2i + AR）;config 单测 - `tests/model_executor/models/mammoth_moda2/test_mammoth_moda2_config.py`;AR 图像理解示例 - `examples/offline_inference/mammothmodal2_preview/`。 -- 已知未决：runner 侧把 hidden 打进 `multimodal_output["latent"]` 的确切 - 代码位、`Mammoth2DecoderLayer` 每层 moe 布线、DiT 去噪逐步数学——分析时 - 未逐行读,改这些区域先补读源码。 From e9bd7c824db6f2a9a49e079e2d653d066cf21944 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 15:52:09 +0800 Subject: [PATCH 272/465] knowledge(vllm-omni): sync 17285c2f (PR #3027) Upstream-Commit: 17285c2f55a41bf15772676121814d59a60ace35 Upstream-PR: #3027 Knowledge-Delta: capture FLUX.2 all-block online FP8 routing, stable prefixes, and unquantized projection boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../repos/vllm-omni/models/flux2/_index.md | 6 +++--- knowledge/repos/vllm-omni/models/flux2/rules.md | 17 +++++++++++++++-- 11 files changed, 29 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a9e1c0a90..79941a20e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 39c16d75a99427e3d8ff04b1d70b6c925475dfc6 + previous_audited_sha: 1e2a6446f79da72b66e15f684d0256244c725916 audited_ref: main - audited_sha: 1e2a6446f79da72b66e15f684d0256244c725916 + audited_sha: 17285c2f55a41bf15772676121814d59a60ace35 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c7d464410..dca76abc6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1e2a6446f79da72b66e15f684d0256244c725916`. The canonical machine baseline +17285c2f55a41bf15772676121814d59a60ace35`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 07aee9e17..02fa9e29b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1e2a6446` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 17285c2f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3452cd024..d61fb9ff2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1e2a6446` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 17285c2f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0f4c14c95..843b3aa61 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1e2a6446` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 17285c2f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7b0a588c2..449cdee4a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1e2a6446` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 17285c2f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d4f6c6b53..b8ae93f85 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1e2a6446` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 17285c2f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index def571d9b..e6744315d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1e2a6446` 验证存在 +- 源码校验：以上路径均已在 `main @ 17285c2f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 563265822..95f8dc7f1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1e2a6446`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 17285c2f`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -88,7 +88,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to 1e2a6446 \ + --to 17285c2f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/flux2/_index.md b/knowledge/repos/vllm-omni/models/flux2/_index.md index 92cd1c703..4d623c41c 100644 --- a/knowledge/repos/vllm-omni/models/flux2/_index.md +++ b/knowledge/repos/vllm-omni/models/flux2/_index.md @@ -1,10 +1,10 @@ --- title: "FLUX.2" created: 2026-07-20 -updated: 2026-07-31 +updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5136", vllm_omni/diffusion/models/flux2/, vllm_omni/diffusion/models/mistral_encoder/] +sources: ["PR #5136", "PR #3027", vllm_omni/diffusion/models/flux2/, vllm_omni/diffusion/models/mistral_encoder/] confidence: high --- @@ -19,6 +19,6 @@ confidence: high ## 什么时候查这里 -- 审查 FLUX.2 text-encoder-only FP8、component prefix、meta 初始化或 CPU offload。 +- 审查 FLUX.2 text-encoder-only FP8、transformer block 的 online FP8 prefix、component prefix、meta 初始化或 CPU offload。 - 描述直达源码与具体不变量见 [FLUX.2 rules](rules.md#direct-代码快速入口)；通用模型适配检查见 [model adaptation guardrails](../../review/guides/model-adaptation-guardrails.md)。 diff --git a/knowledge/repos/vllm-omni/models/flux2/rules.md b/knowledge/repos/vllm-omni/models/flux2/rules.md index 2aac6406c..382069cd5 100644 --- a/knowledge/repos/vllm-omni/models/flux2/rules.md +++ b/knowledge/repos/vllm-omni/models/flux2/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4645", "PR #5136", vllm_omni/diffusion/models/flux2/flux2_transformer.py, "PR #5910"] +sources: ["PR #4645", "PR #5136", vllm_omni/diffusion/models/flux2/flux2_transformer.py, "PR #5910", "PR #3027"] confidence: high --- @@ -16,7 +16,7 @@ confidence: high | PR 描述信号 | 规则组 | 第一批源码 | |---|---|---| -| FLUX.2 pipeline/transformer | FLUX2-1b | `diffusion/registry.py::_DIFFUSION_MODELS["Flux2Pipeline"]` → `diffusion/models/flux2/pipeline_flux2.py::Flux2Pipeline`、`flux2_transformer.py::Flux2Transformer2DModel` | +| FLUX.2 pipeline/transformer | FLUX2-1b、FLUX2-1c | `diffusion/registry.py::_DIFFUSION_MODELS["Flux2Pipeline"]` → `diffusion/models/flux2/pipeline_flux2.py::Flux2Pipeline`、`flux2_transformer.py::Flux2Transformer2DModel` | | Mistral text encoder、FP8、component namespace | FLUX2-1a | `diffusion/models/mistral_encoder/` → component quantization selector/loader | | meta parameter、CPU offload、BF16 baseline | FLUX2-1b | FLUX.2 component loader/materialization 路径 → `Flux2Pipeline` | | SP auto-padding、`mask_sp_padding`、dense/varlen | [SPPAD-1a](../../components/diffusion/rules-sp-padding.md#sppad-1a-padding-mask-是显式正确性性能策略不是无损优化) | Flux2 `_sp_plan` → forward context → `hidden_states_mask` → attention | @@ -43,5 +43,18 @@ FLUX.1 和 FLUX.2-Klein 不自动归到本页；必须由描述/registry key 明 - 验收：meta parameter 完成量化物化后再迁移；运行与测试完全一致的 10-step case， LPIPS 阈值由该 case 直接产生。 ^[PR #5136] +## FLUX2-1c — transformer online FP8 必须到达两类 block，外层 projection 保持 BF16 + +- 触发：修改 FLUX.2 transformer 的 online FP8 构造、`quant_config`、block prefix 或量化覆盖范围。 +- 强制：将同一 online `quant_config` 传给每个 `Flux2TransformerBlock` 与 + `Flux2SingleTransformerBlock`，并使用稳定的 `transformer_blocks.` 与 + `single_transformer_blocks.` prefix 供 component selector 匹配；`x_embedder`、 + `context_embedder` 与 `proj_out` 保持普通 BF16 `nn.Linear`。 +- 禁止：用仅适用于 serialized/ModelOpt checkpoint 的 safety config 覆盖 double-stream + block，从而关闭 online FP8；或因同属 transformer 就量化上述三个 outer projection。PR + 历史中的早期 H20 benchmark 来自已删除实现，不得当作当前 main 的性能或质量证据。 +- 验收：用 dynamic `Fp8Config` 构造小模型，分别枚举两类 block 的 config 与 prefix，并断言 + 三个 outer projection 仍是 `nn.Linear`。 ^[PR #3027] + 共享 component-quantization 合同见 [Diffusion rules](../../components/diffusion/rules.md)； 加载器 upstream 边界见 [loading drift](../../rebase/upstream-api-drift-loading.md)。 From eb9a138f069fd495524f28ee48bf4ef38bdcd350 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 15:57:41 +0800 Subject: [PATCH 273/465] knowledge(vllm-omni): sync 1c917b27 (PR #5687) Upstream-Commit: 1c917b275148fa6427ecf9eabb46bbd6df0f589e Upstream-PR: #5687 Knowledge-Delta: capture Qwen3-Omni one-time AWQ name mapping and recursive component-config remapping --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../repos/vllm-omni/models/qwen-omni/_index.md | 8 ++++++-- .../repos/vllm-omni/models/qwen-omni/rules.md | 17 ++++++++++++++++- 11 files changed, 33 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 79941a20e..d2e3d7500 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1e2a6446f79da72b66e15f684d0256244c725916 + previous_audited_sha: 17285c2f55a41bf15772676121814d59a60ace35 audited_ref: main - audited_sha: 17285c2f55a41bf15772676121814d59a60ace35 + audited_sha: 1c917b275148fa6427ecf9eabb46bbd6df0f589e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index dca76abc6..ae0bb22a8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -17285c2f55a41bf15772676121814d59a60ace35`. The canonical machine baseline +1c917b275148fa6427ecf9eabb46bbd6df0f589e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 02fa9e29b..cb6d2cb91 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 17285c2f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1c917b27` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d61fb9ff2..f5e9ec33d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 17285c2f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 1c917b27` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 843b3aa61..913ab4796 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 17285c2f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1c917b27` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 449cdee4a..30c1e89dd 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 17285c2f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1c917b27` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b8ae93f85..40f6bfe04 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 17285c2f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1c917b27` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e6744315d..75022a839 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 17285c2f` 验证存在 +- 源码校验：以上路径均已在 `main @ 1c917b27` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 95f8dc7f1..7f0c585bc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 17285c2f`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1c917b27`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -88,7 +88,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to 17285c2f \ + --to 1c917b27 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index 5034be3fb..d8fce4893 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: index tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", "PR #5671", "PR #5976", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, docs/design/qwen3_omni_tts_performance_optimization.md] +sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/quantization/component_config.py, docs/design/qwen3_omni_tts_performance_optimization.md] --- # Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS） @@ -24,6 +24,10 @@ sources: ["PR #5073", "PR #5671", "PR #5976", vllm_omni/model_executor/models/qw - Qwen3-Omni 的 MUSA ModelOpt FP8 边界是 Thinker 保留量化、Talker/Code2Wav 通过 deploy overlay 显式清除 root quantization metadata，并仅关闭不安全的 Talker-MTP FULL graph wrapper；精确三态与验证边界归下方 architecture 入口。 +- Qwen3-Omni 的 AWQ/compressed-tensors checkpoint 名称必须在统一 wrapper 映射一次；嵌套 + Thinker/Talker/Code2Wav 不得再次改名，仅在自身声明 packed modules 时补充对应 metadata。 + component/default quant configs 都是同一映射合同的一部分；验收规则归本目录的 + Qwen-Omni rules。^[PR #5687] - vLLM 0.27 后 upstream 与 Omni registry 有同名 architecture：Omni 必须覆盖全局 entry；plain `vllm serve` 缺 `model_stage` 时 Qwen2.5/Qwen3 默认 thinker，Qwen3 thinker 在非 staged 模式返回 bare tensor，只有 staged talker consumer 才请求 captured layers；共享验收见 @@ -47,4 +51,4 @@ sources: ["PR #5073", "PR #5671", "PR #5976", vllm_omni/model_executor/models/qw | 遇到什么 | 查看哪里 | |---|---| | stage 拓扑、代际差异与官方性能优化结论 | [architecture](architecture.md) | -| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary 或固定种子音频回归 | [Qwen-Omni rules](rules.md) | +| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary、AWQ/compressed-tensors 名称映射或固定种子音频回归 | [Qwen-Omni rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index 111c71b3d..903052056 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, models, qwen-omni] -sources: ["PR #6449", vllm_omni/deploy/qwen3_omni_moe.yaml, tests/config/test_config_factory.py] +sources: ["PR #5687", "PR #6449", vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py] confidence: high --- @@ -16,3 +16,18 @@ confidence: high - 强制：CUDA platform overlay 仅为 Thinker stage 0 设置 `compilation_config.custom_ops: ["+rotary_embedding"]`，保持 MRoPE 在 CUDA/Triton custom-op 边界后；Talker/Code2Wav 与 CPU、MUSA、NPU、ROCm、XPU 配置不得继承此 override。 - 禁止：以全局 `enforce_eager` 取代窄 graph boundary；启用更宽的 Q/K norm-RoPE fusion flag；只检查 YAML 文本，或将 Thinker 数值稳定性外推为全部 Qwen stage/平台。 - 验收：经 `_apply_platform_overrides → merge_pipeline_deploy` 断言 CUDA 仅 stage 0 有该 custom op、其余 stage/平台均无；固定 seed 下与 eager oracle 的 thinker/talker/codec 离散轨迹必须完全一致，并验证生成 WAV 逐字节一致。相关配置展开见 [Configuration rules](../../components/configuration/rules.md)，家族 topology 见 [architecture](architecture.md)。^[PR #6449] + +## QOMNI-1b — Qwen3-Omni 嵌套 stage 的量化名称只能在统一 wrapper 映射一次 + +- 触发：修改 Qwen3-Omni 的 `SupportsQuant`、`WeightsMapper`、compressed-tensors/AWQ + ignore/exclude 名称，或 Thinker、Talker、Code2Wav 的嵌套模型构造。 +- 强制：在统一 Qwen3-Omni wrapper 暴露 checkpoint 到最终嵌套模块路径的映射，并先对完整 + quantization config 执行一次映射；`ComponentQuantizationConfig` 必须递归映射每个非空 + component 和 default config。子 stage 检测到该映射后不得再次改写名称，但仍须合并自己的 + `packed_modules_mapping`，供量化 scheme 匹配 fused module。 +- 禁止：只映射一个 component wrapper 而给它标记为已映射；让 Thinker/Talker/Code2Wav 各自 + 重复应用 stage mapper；因修复名称匹配而把 BF16 router/encoder 也纳入量化；或把该构造期 + 映射修复描述成 quantization kernel / inference hot path 变化。 +- 验收：以真实 component-wrapped `CompressedTensorsConfig`（含 `None` component 与 default） + 断言 `ignore` HF names 映射到最终路径；加载 AWQ checkpoint 时三 stage 都能完成 weight + loading，且不出现缺模块/参数或 routed-expert quantized-weight 属性错误。^[PR #5687] From b24957a6d2b46efa9a2ae2b1935674bfb4f9f862 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 16:10:07 +0800 Subject: [PATCH 274/465] knowledge(vllm-omni): sync 3d035bfa (PR #6476) Upstream-Commit: 3d035bfa190e303f53d72e3baa10885f60abe682 Upstream-PR: #6476 Knowledge-Delta: capture transactional dynamic LoRA activation and MiniMax-H3 Turbo artifact, sampling, and offload contracts --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules-lora.md | 12 ++++++++++-- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../repos/vllm-omni/models/minimax-h3/_index.md | 7 +++++-- .../models/minimax-h3/rules-cache-task.md | 14 +++++++++++--- .../repos/vllm-omni/models/minimax-h3/rules.md | 4 ++-- 13 files changed, 39 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d2e3d7500..539342697 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 17285c2f55a41bf15772676121814d59a60ace35 + previous_audited_sha: 1c917b275148fa6427ecf9eabb46bbd6df0f589e audited_ref: main - audited_sha: 1c917b275148fa6427ecf9eabb46bbd6df0f589e + audited_sha: 3d035bfa190e303f53d72e3baa10885f60abe682 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ae0bb22a8..9256223b9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1c917b275148fa6427ecf9eabb46bbd6df0f589e`. The canonical machine baseline +3d035bfa190e303f53d72e3baa10885f60abe682`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index cb6d2cb91..221ac1d75 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1c917b27` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3d035bfa` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f5e9ec33d..74b422149 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1c917b27` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 3d035bfa` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md index 15ebb932d..d4af11132 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070"] +sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070", "PR #6476", vllm_omni/diffusion/models/minimax_h3/lora.py, tests/diffusion/lora/test_lora_manager.py] confidence: high --- @@ -20,10 +20,11 @@ confidence: high | `--lora-backend`、startup path/scale、deploy/CLI projection | `DIFF-2l` | `entrypoints/cli/serve.py` → `async_omni_engine.py`/stage config → `OmniDiffusionConfig` → `DiffusionWorker.init_lora_manager` | | distilled merge、key conversion、packed QKV、alpha、unload | `DIFF-2m` | `diffusion/lora/loader.py` → pipeline mixin → transformer parameter | | Wan dual transformer、high/low-noise file order、partial load | `DIFF-2n` | `WanLoraLoaderMixin` → `get_transformer_from_pipeline(name)` → `transformer`/`transformer_2` | +| legacy dynamic adapter hook、packed/stacked binding、activation rollback | `DIFF-2x` | `DiffusionLoRAManager::{_load_adapter,_bind_adapter_weights,_activate_adapter}` → model pipeline hook | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| -| `checkpoint-distributed` | diffusion LoRA backend、checkpoint、融合与 transformer mapping | `DIFF-2l`, `DIFF-2m`, `DIFF-2n` | +| `checkpoint-distributed` | diffusion LoRA backend、checkpoint、融合与 transformer mapping | `DIFF-2l`, `DIFF-2m`, `DIFF-2n`, `DIFF-2x` | ## DIFF-2l — backend 是 startup 配置，不是 request 字段 @@ -90,3 +91,10 @@ confidence: high - 禁止：为 phase adapter 增加第二个 resident Transformer 或 LTX 专用环境变量；允许 request/static LoRA 与内部 phase adapter 组合；在非 BF16 或量化 base 上声称 layer-fused 可用；对缺失配对、未映射模块或不匹配 shard 静默跳过。 - 验收：覆盖官方 key mapping、缺失 A/B、unmapped/duplicate target、dynamic 与 fused 精确算术、Row/Column/QKV 本地切片、base 权重不变、phase 进入/退出和 finalize 时机；同时验证 generic `model_paths` sidecar 覆盖与 model-root/Hub fallback。^[PR #5500] +## DIFF-2x — legacy dynamic manager 的模型专用 loader 与 activation 必须 fail-closed 且事务化 + +- 触发：修改 `DiffusionLoRAManager` 的 request-time adapter load/bind、模型专用 loader hook 或 packed/stacked target。 +- 强制：先让 pipeline 的可选 loader 识别并返回其模型拥有的 `(LoRAModel, PEFTHelper)`，仅在其返回 `None` 时走既有 generic PEFT fallback；专用 loader 的 artifact、target 和全局 A/B shape 必须在 wrapper mutation 前完成校验。manager 必须记录实际绑定的 logical weights，模型 validator 证明完整覆盖；packed QKV/FFN 等布局由模型明确声明或转换，不由共享 manager 猜测。 +- 强制：activation fast path 只可在 active ID 与 scale 都匹配时跳过。首次 mutation 前清除 active ID；任一 `set_lora()` 或 binding validator 异常都 reset 所有 wrapper slot 并保持 inactive，不能保留混合权重或陈旧 fast-path 状态。 +- 禁止：将已识别的专用 artifact 悄悄退回 generic loader；按名称而非真实绑定确认成功；将一个模型的 packed layout/scale 规则泛化给所有 diffusion 模型；失败后继续声明旧 adapter active。 +- 验收：CPU regressions 覆盖 model hook/fallback、binding completeness、mid-loop failure 后 reset 与旧 adapter retry；模型 integration 另覆盖 artifact identity、target/shape rejection、packed QKV/FFN binding 和实际 request sampling contract。^[PR #6476] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 913ab4796..42b15619c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1c917b27` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3d035bfa` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 30c1e89dd..19a015322 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1c917b27` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3d035bfa` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 40f6bfe04..a47808ad5 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1c917b27` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3d035bfa` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 75022a839..0b873792e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1c917b27` 验证存在 +- 源码校验：以上路径均已在 `main @ 3d035bfa` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 7f0c585bc..e1b52f257 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1c917b27`（2026-08-23）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3d035bfa`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -88,7 +88,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to 1c917b27 \ + --to 3d035bfa \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 9015e9fc4..3b95990b2 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] confidence: high --- @@ -72,6 +72,9 @@ confidence: high 交互与单 H100 校准边界见 MMH3-2g。 - distilled checkpoint 的 continuous RF `base_schedule`、四步语义与 partition 隔离见 MMH3-2h；它不替换 H3 的自定义 solver。 +- LightX2V Turbo v1.0 仅支持 legacy dynamic LoRA；精确 artifact mapping、packed QKV/FC1 + binding、active-request sampling 合同与 offload 排除项见 MMH3-2j；转换后的 mixed-rank + native PEFT checkpoint 不在该支持声明内。 - datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；packed suffix trim、SAGE 短序列 role、 mask/metadata fail-closed、AllGather-KV 禁用及性能证据边界见 MMH3-1e。 - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention @@ -111,4 +114,4 @@ T2VA 因当前严格 gate 未达标被移出；这表示“无 retained pixel-le H3 input matrix/media ingress，以及 text-encoder completeness、online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 [media rules](rules-media.md) 与 [MiniMax H3 rules](rules.md#direct-代码快速入口)；checkpoint transform、quantized loader 与 text-encoder fused-source 完整性正文见 [loading rules](rules-loading.md)；DLO、consumer/H100/ROCm -部署和硬件证据正文见 [deployment rules](rules-deployment.md)；conditioned VAE 确定性、modular task 选择与 request 级 Cache-DiT/TeaCache/sigma schedule 生命周期见 [缓存与任务生命周期规则](rules-cache-task.md)。 +部署和硬件证据正文见 [deployment rules](rules-deployment.md)；conditioned VAE 确定性、modular task 选择与 request 级 Cache-DiT/TeaCache/sigma schedule/Turbo LoRA 生命周期见 [缓存与任务生命周期规则](rules-cache-task.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md index af648c5f6..fe9506833 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md @@ -4,14 +4,14 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5720", "PR #5837", "PR #5840", "PR #5853", "PR #5991", vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py] +sources: ["PR #5703", "PR #5720", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py] confidence: high --- # MiniMax H3 缓存与任务生命周期规则 -`MMH3-2c`–`MMH3-2h`：conditioned VAE 的确定性、modular task 选择，以及 request 级 -Cache-DiT、TeaCache 与 distilled sigma schedule 的生命周期。触发信号见 +`MMH3-2c`–`MMH3-2j`：conditioned VAE 的确定性、modular task 选择，以及 request 级 +Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期。触发信号见 [MiniMax H3 规则](rules.md) 的 Direct 代码快速入口；加载合同见 [加载规则](rules-loading.md)，媒体输入见 [媒体规则](rules-media.md)。 @@ -154,6 +154,14 @@ Cache-DiT、TeaCache 与 distilled sigma schedule 的生命周期。触发信号 matching/mismatched explicit step 和 legacy/partially-constructed fallback。仍缺真实 distilled checkpoint 的 CUDA/NPU E2E 与 teacher/student 质量阈值。^[PR #5991] +## MMH3-2j — Turbo LoRA 只接受精确发布 artifact，并与 H3 task、sampling 和 offload lifecycle 共同门禁 + +- 触发：MiniMax-H3 LightX2V Turbo 的 dynamic LoRA loader、artifact conversion、request task/steps/flow shift，或与 CPU/layerwise/distributed-layerwise offload 的组合。 +- 强制：仅识别声明文件名的 LightX2V Turbo v1.0 safetensors，且 metadata 必须为 `key_format=minimax-h3-diffusers`、rank/alpha `128/128`；任何未知 tensor、重复/不完整 A/B pair、unsupported target 或不精确的 global A/B shape 都在 binding 前失败。转换必须把 Diffusers token-refiner/main-block names 映射为 native H3 names，恢复 FC1 `[gate; up]` 行序为 model-owned packed weights，并通过 `stacked_params_mapping` 将独立 Q/K/V 接到 packed QKV。 +- 强制：Turbo 仅在非零 scale 的实际 active recognized adapter 时限制请求；只支持 FL2VA/T2VA，要求五个 sigma points（四次 denoiser evaluation）、video `flow_shift=6` 与 audio `audio_flow_shift=3`。每次真实 load 都替换该 client ID 的 Turbo classification，避免 eviction 后同 ID generic adapter 被误分类。 +- 禁止：将 exact Turbo artifact 的损坏 metadata 静默交给 generic fallback；将任意 H3 PEFT checkpoint、prefusion、multi-LoRA composition 或 Ref2VA 宣称为此功能支持；在 model-level CPU、layerwise 或 distributed-layerwise offload 下激活 Turbo，因为 legacy wrapper 的 LoRA tensors 不随 base parameter lifecycle 搬运。 +- 验收：覆盖 file/metadata/alpha/target/pair/shape rejection、QKV and FC1 packing、scale-zero/generic same-ID/Ref2VA lifecycle、steps/shifts errors 与三种 offload rejection；真实 artifact evidence 必须另绑定 exact checkpoint、task、topology、sampling 和 video/audio output。转换后的 native mixed-rank PEFT artifact 不属于本 PR 的 supported contract；其 generic fallback rank/scale mismatch 是 post-merge follow-up，不可用本页的 Turbo success 证明安全。^[PR #6476] + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 0f93dd894..69929194f 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476"] confidence: high --- @@ -32,6 +32,7 @@ confidence: high | force-refresh hint、once/repeat、reinstall key | [MMH3-2f](rules-cache-task.md#mmh3-2f-force-refresh-hint-属于-active-profile-identity) | H3 `extra_args` validation → immutable cache config → installation key/refresh context | | TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | [MMH3-2g](rules-cache-task.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state) | custom enabler → H3 extractor → module-resident hook state / per-generation reset | | distilled、DMD2、`base_schedule`、4-step、sigma boundary | [MMH3-2h](rules-cache-task.md#mmh3-2h-distilled-sigma-schedule-按-partition-所有且以区间计步) | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | +| LightX2V Turbo、legacy dynamic LoRA、packed QKV/FC1、five sigma points | [MMH3-2j](rules-cache-task.md#mmh3-2j-turbo-lora-只接受精确发布-artifact并与-h3-tasksampling-和-offload-lifecycle-共同门禁) | `lora.py::load_minimax_h3_turbo_lora` → pipeline hook/sampling validation → `DiffusionLoRAManager` | | DLO、TP-local、resident layers、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | @@ -207,4 +208,3 @@ confidence: high - 强制：`MiniMaxH3MLP` 必须把 `fc1` 的 packed 输出按 `[gate, up]` 顺序直接交给本地 `SiluAndMul`，再将结果交给既有 `fc2`；模型接入遵循 [DIFF-1v](../../components/diffusion/rules.md) 的平台与精度合同，且只改变 H3 DiT，不改变 Qwen3-VL text encoder 的独立 MLP 路径。 - 禁止：在已选择 fused 路径后重新 `chunk` 并执行两次 native SiLU/乘法；用通用 vLLM activation 类替代本地 diffusion layer；把当前 NPU native fallback 描述成 H3 NPU fused support，或把一次 H3 性能/精度观察推广到其他 task、平台和拓扑。 - 验收：H3 contract test 用 packed sentinel 验证 gate/up 顺序、半维输出及 `fc2` 输入，并分别覆盖 CUDA fused 与 NPU/XPU/native fallback；固定 checkpoint、seed、steps 和输入对照 native/fused 输出并设数值容差，同时以 kernel/profile 证据确认优化路径确实生效。^[PR #6283] - From efbd6518b5f96740f92ab2d9a4802885a15ec5e7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 16:18:44 +0800 Subject: [PATCH 275/465] knowledge(vllm-omni): sync 072bfc02 (PR #4820) Upstream-Commit: 072bfc02dd74cb0eb5c2f2a914e5dbbddba43b65 Upstream-PR: #4820 Knowledge-Delta: capture FastVideo VSA routing, checkpoint-gated Wan integration, and fixed DMD sampling --- adapters/vllm_omni/release_baseline.yaml | 8 ++++---- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/configuration/rules.md | 14 +++++++++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules.md | 16 +++++++++++++++- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 9 ++++++--- .../repos/vllm-omni/models/wan2-2/_index.md | 10 ++++++---- .../vllm-omni/models/wan2-2/architecture.md | 11 ++++++++++- 13 files changed, 61 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 539342697..51ef07166 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1c917b275148fa6427ecf9eabb46bbd6df0f589e + previous_audited_sha: 3d035bfa190e303f53d72e3baa10885f60abe682 audited_ref: main - audited_sha: 3d035bfa190e303f53d72e3baa10885f60abe682 + audited_sha: 072bfc02dd74cb0eb5c2f2a914e5dbbddba43b65 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 81 sha256: 9d5a685eae7ef099bb884557ca51b0e1b5f19c8555aa16bbd136c2081d95cd78 diffusion: - count: 64 - sha256: 6bebd607c9e9570f6c4622d220a9a7db6f5d07e3d008b1adb6ee526f68af49b8 + count: 65 + sha256: 5d50c8f9cc5aeb93ab1c6b9e2ba8eca145cb05f8343eef064e2a5ab873dfc556 pipelines: count: 55 sha256: 9d5033c9a7ffd7ff636e32db0f9be2a549b7c16b034e13817e280f19a9e16769 diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9256223b9..7779efe0a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3d035bfa190e303f53d72e3baa10885f60abe682`. The canonical machine baseline +072bfc02dd74cb0eb5c2f2a914e5dbbddba43b65`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 221ac1d75..553da2bd9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3d035bfa` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 072bfc02` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 08b1b9ccd..a6faf07c0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820"] --- # vLLM-Omni 配置开发门禁 @@ -229,3 +229,15 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 强制：将 registry 所需的静态 `PipelineConfig` 放入无 runtime 副作用的轻量模块；registry 只导入该模块，runtime pipeline 继续 re-export 同一对象以保持既有导入路径，并用干净子进程验证 registry 解析不加载 runtime。 - 禁止：让 `pipeline_registry` eager import 模型 runtime、依赖已预加载模块或导入顺序掩盖循环；不得因拓扑模块轻量就改变模型 runtime 的实际加载语义。 - 验收：隔离进程中导入并解析目标 registry key，断言对应 runtime module 不在 `sys.modules` 且返回的 `PipelineConfig` 身份与字段正确；再验证 runtime 的兼容导入路径仍 re-export 同一拓扑对象。^[PR #6322] + +### VOMNI-CFG-2c — backend 专用参数必须有完整 typed reachability + +- 触发：为 diffusion attention backend 新增 CLI、环境、deploy 或 structured config 参数。 +- 强制：参数必须在 `StageDeployConfig`、orchestrator/CLI、normalization、`AttentionSpec` 和 + `backend_kwargs` 间保持同一 canonical 值，并验证从每个公开入口到 backend consumer；只允许 + 相应 backend 接受，且在无法实际稀疏执行的 shape/Top-K 组合显式记录或验证 fallback。 +- 禁止：只把字段加到 schema、只在 CLI 解析，或让 deploy projection 静默丢弃；不得把 backend + 专用值作为未经类型校验的任意 dict 透传。 +- 验收：覆盖有效非默认值的 CLI、deploy/structured 和 direct attention-config 路径，断言最终 + `backend_kwargs`；覆盖错误 backend、非正值，以及 Top-K 超过 runtime block count 的回退。 + `WanDMDPipeline` 一类 checkpoint alias 还必须由 `model_index.json` 自动发现。^[PR #4820] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 74b422149..fef1407dc 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3d035bfa` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 072bfc02` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 4108b86f6..f3aa38c61 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py] confidence: high --- @@ -221,3 +221,17 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 强制：发现 `submod._omni_original_forward` 时只编译该原始 compute callable；保留 `submod.forward` 作为 hook dispatcher，将 `_omni_original_forward` 重绑定到 compiled callable，并更新仍指向编译前 callable 的 hook `fn_ref.original_forward`。 - 禁止：把 `HookRegistry` dispatcher 或其 stream/event/storage/offload 控制流编译进 regional graph；直接覆盖 `forward`；让 hook 保留过期的 eager callable。 - 验收：测试断言 `torch.compile` 收到原始 callable、`forward` 仍为 wrapper、hook reference 指向 compiled callable，且 block 调用使用编译结果；DLO+regional compile smoke 还须覆盖有限输出与首次 lazy denoise 不崩溃。^[PR #6073] + +### DIFF-10b — 稀疏 video attention 必须先证明 kernel 前提，否则可观测地回退 + +- 触发：新增或修改 video DiT sparse-attention backend、tile metadata、kernel custom op 或其 + compile boundary。 +- 强制：backend 必须验证 CUDA、FP16/BF16、self-attention `[B,S,H,D]`、相同 Q/K/V + shape/dtype、无 mask/GQA/SP、合法 softmax scale，以及 `vsa_dit_seq_shape` 与 flattened + sequence 的乘积一致；block geometry 必须是 kernel 支持的 256 tokens，Top-K 不得超过 + runtime block count。外部 CUDA op 以 custom-op 边界留在 `torch.compile` 图外。 +- 禁止：仅凭选择了 backend 就声称走稀疏 kernel；从 flat sequence 猜三维 grid；在不支持时 + 静默改变 shape、dtype 或 attention 语义。 +- 验收：unit 覆盖每个关键拒绝条件及 SDPA fallback，并以真实 Wan grid metadata 覆盖 tile/ + untile 回排。`topk == num_blocks` 的 native Wan 走 SDPA，FastVideo DMD checkpoint 可显式 + 保留 all-block VSA；两条路由都须在日志或测试中可辨。^[PR #4820] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 42b15619c..c3122a7b0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3d035bfa` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 072bfc02` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 19a015322..2348c0157 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3d035bfa` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 072bfc02` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a47808ad5..ab5eee9e8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3d035bfa` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 072bfc02` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0b873792e..519d85e7b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3d035bfa` 验证存在 +- 源码校验：以上路径均已在 `main @ 072bfc02` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e1b52f257..4f79c329a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3d035bfa`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 072bfc02`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -36,7 +36,7 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | -| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 64 条 pipeline / 41 个模型族目录 | +| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 41 个模型族目录 | | Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 55 个 key | | Deploy YAML | `vllm_omni/deploy/*.yaml` | 84 份 | @@ -47,6 +47,9 @@ architecture、两个 pipeline key、两个 deploy YAML 及其 tokenizer/config/ 因此 active owner 与 catalog 不再声明这两个家族；diffusion text-to-audio 的 first-party 示例只剩 Stable Audio，Audex 的 TTA 仍是独立两阶段 autoregressive 路径。^[PR #6353] +PR #4820 将 `WanDMDPipeline` 注册为 Wan2.2 的同实现入口，因此 diffusion pipeline +计数回升至 65；它不是新的模型族或新的 `OMNI_PIPELINES` key。^[PR #4820] + ## AR/omni 模型族（31） aura_omni、audex、bagel、cosyvoice3、covo_audio、dots_tts、dynin_omni、fish_speech、 @@ -88,7 +91,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to 3d035bfa \ + --to 072bfc02 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/wan2-2/_index.md b/knowledge/repos/vllm-omni/models/wan2-2/_index.md index 7592b956f..decf70834 100644 --- a/knowledge/repos/vllm-omni/models/wan2-2/_index.md +++ b/knowledge/repos/vllm-omni/models/wan2-2/_index.md @@ -1,10 +1,10 @@ --- -title: "Wan 2.2（六架构视频家族:T2V/I2V/VACE/S2V/DMD2×2）" +title: "Wan 2.2（七个 registry 架构名:T2V/I2V/VACE/S2V/DMD2×2/FastVideo DMD）" created: 2026-07-21 updated: 2026-09-02 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #2783", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/wan2_2/, vllm_omni/deploy/wan2_2_ti2v.yaml, vllm_omni/diffusion/models/dmd2/mixin.py, tests/diffusion/lora/test_loader.py] +sources: ["PR #2783", "PR #4820", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/wan2_2/, vllm_omni/deploy/wan2_2_ti2v.yaml, vllm_omni/diffusion/models/dmd2/mixin.py, tests/diffusion/lora/test_loader.py] --- # Wan 2.2 @@ -22,7 +22,8 @@ sources: ["PR #2783", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/mo `WanVACEPipeline`→`pipeline_wan2_2_vace.Wan22VACEPipeline`; `WanS2VPipeline`→`pipeline_wan2_2_s2v.Wan22S2VPipeline`; `WanT2VDMD2Pipeline`（同模块于 pipeline_wan2_2）与 - `WanI2VDMD2Pipeline`（pipeline_wan2_2_i2v）——**六个都有 pre+post + `WanI2VDMD2Pipeline`（pipeline_wan2_2_i2v）与 `WanDMDPipeline`（映射到 + `Wan22Pipeline`）——**七个都有 pre+post process 绑定**（DMD2 复用基类函数）。 - 入口路径：registry `vllm_omni/diffusion/registry.py` 与 `vllm_omni/config/pipeline_registry.py`;拓扑 @@ -35,7 +36,8 @@ sources: ["PR #2783", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/mo （Wan2.1 式 checkpoint）;VACE = 参考图/源视频/掩码条件,单或双专家形态都 接受;S2V = 单 transformer,支持 diffusers 与原始格式 checkpoint （T5→UMT5 转换）;两个 DMD2 = 分别继承 T2V/I2V 行为,换 DMD2 调度并禁 - CFG。 + CFG；`WanDMDPipeline` 是 FastVideo checkpoint 的三步 flow path，不与 DMD2 mixin + 混同。 - import 期副作用：`__init__.py` 调 `patch_wan_rms_norm()` 把所有已加载 diffusers 模块里的 `WanRMS_norm` 换成本仓 `RMSNormVAE`——**进程内任何 diffusers Wan VAE 用户都被影响**（例如 diff --git a/knowledge/repos/vllm-omni/models/wan2-2/architecture.md b/knowledge/repos/vllm-omni/models/wan2-2/architecture.md index 371d57dd0..b62f5f8e8 100644 --- a/knowledge/repos/vllm-omni/models/wan2-2/architecture.md +++ b/knowledge/repos/vllm-omni/models/wan2-2/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-21 updated: 2026-09-02 type: architecture tags: [vllm-omni, models, diffusion] -sources: ["PR #2783", "PR #5969", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py, vllm_omni/diffusion/models/dmd2/mixin.py] +sources: ["PR #2783", "PR #5969", "PR #4820", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py, vllm_omni/diffusion/models/dmd2/mixin.py] --- # Wan 2.2 架构 @@ -43,6 +43,15 @@ diffusion 设施见 [Diffusion 组件](../../components/diffusion/_index.md)。 `_sanitize_dmd2_request` 强制步数/guidance、清负向 prompt、**并从 extra_args 里剥掉 `sample_solver`/`flow_shift`**——防止基类把 scheduler 换回去。给 DMD2 变体"加回 CFG/负向"的 PR 违反此合同。 +- **FastVideo WanDMD 合同**：checkpoint 的 `_class_name="WanDMDPipeline"` 以 + `diffusers_class_aliases` 发现后仍复用 `Wan22Pipeline`，但固定 Euler、shift `8.0`、 + timesteps `(1000, 757, 522)`，并把 guidance 缺省值改为 `1.0`；它忽略 request 的步数（包括 engine + warmup）和 solver/shift override，并在每一步按 `predict_clean → add_noise(next timestep)` + 推进。这与 DMD2 mixin 是两套调度契约，不能合并或相互覆盖。所有 Wan T2V/I2V/S2V/VACE + loader 都通过 shared helper：checkpoint 缺少 `to_gate_compress` 时删除零初始化 projection， + 有权重时保留 learned gate；这项 loader 清理不代表这些变体都受 VSA 正式支持。当前正式范围 + 仅是 `FastVideo/FastWan2.2-TI2V-5B-Diffusers` 通过 `Wan22Pipeline` 提供的 T2V/I2V 模式， + 独立 I2V-14B、S2V 与 VACE pipeline 均在范围外。^[PR #4820] - 量化:每 transformer 独立解析 `quantization_config`,双专家可各自量化。 - distilled LoRA 同样按 architecture-declared transformer ownership 分开：T2V/I2V 及其 DMD2 子类的 mixin 读取 `has_transformer_2`，要求位置 0/1 分别对应 `transformer`/`transformer_2`， From 0485dfa32c96234911165b1fd285f6bc6d426b90 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 16:25:38 +0800 Subject: [PATCH 276/465] knowledge(vllm-omni): sync d15b7ca3 (PR #6084) Upstream-Commit: d15b7ca3e12ce969a37547f7480470b5b755dbf3 Upstream-PR: #6084 Knowledge-Delta: capture AR lifecycle routing, admission gating, and acknowledged abort cleanup --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../serving/rules-engine-lifecycle.md | 23 ++++++++++++++++++- .../vllm-omni/components/serving/rules.md | 4 ++-- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 11 files changed, 35 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 51ef07166..4d26c9b64 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3d035bfa190e303f53d72e3baa10885f60abe682 + previous_audited_sha: 072bfc02dd74cb0eb5c2f2a914e5dbbddba43b65 audited_ref: main - audited_sha: 072bfc02dd74cb0eb5c2f2a914e5dbbddba43b65 + audited_sha: d15b7ca3e12ce969a37547f7480470b5b755dbf3 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7779efe0a..6b24bcc5e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -072bfc02dd74cb0eb5c2f2a914e5dbbddba43b65`. The canonical machine baseline +d15b7ca3e12ce969a37547f7480470b5b755dbf3`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 553da2bd9..f30583b7c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 072bfc02` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d15b7ca3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index fef1407dc..2957e8a4c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 072bfc02` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d15b7ca3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c3122a7b0..41c167cd1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 072bfc02` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d15b7ca3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2348c0157..7f275c4b9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 072bfc02` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d15b7ca3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ab5eee9e8..047d3c739 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 072bfc02` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d15b7ca3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 519d85e7b..a7518f58f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 072bfc02` 验证存在 +- 源码校验：以上路径均已在 `main @ d15b7ca3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 5750b0ad4..29dca0b3d 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346"] confidence: high --- @@ -135,6 +135,27 @@ confidence: high - 禁止：让 server 保存或修改静态能力；把上传 voice 合并进 built-in speaker 集合后在删除时 `discard`，或把 frozen dataclass 误解为内部 profile mapping 的深层不可变；在共享 serving 中恢复 model-type capability 分支。 - 验收：构造相关 adapter，断言能力在初始化后进入 snapshot、集合类型正确且内置/预计算/上传 voice 的并集完整；删除与 built-in 同名的上传 voice 后内置 voice 仍存在，并验证无 adapter 的 diffusion 路径不会触发 adapter 属性错误或专属维度校验。^[PR #6138] +### SERV-5m — AR lifecycle 控制必须先封锁 admission，并以 backend 完成信号收尾 + +- 触发：修改 `AsyncOmni` 的 pause/resume、sleep/wake 或 abort，或改变 + `StagePool.collective_rpc` 到 AR EngineCore / diffusion worker 的控制面路由。 +- 强制：在 sleep drain/offload 前设置 frontend admission gate；AR stage 经 orchestrator + 路由到 EngineCore 的 `pause_scheduler`、`resume_scheduler`、`sleep`、`wake_up`，diffusion + stage 继续走 worker sleep/wake RPC。`wake_up()` 不得替 AR/mixed engine 解除 admission，只有 + `resume_generation()` 可以解除；已 paused 时的定向 pause 和 cache clear 仍须执行。stage ID + 必须先校验范围。只允许这四个已知 EngineCore helper 使用 `*_async` fast path，并保持 caller + timeout；其余 RPC 仍走 collective path。需要 frontend cleanup 的 abort 必须关联 result，等待 + orchestrator 完成 stage abort、binding release 和 request cleanup 后才移除 frontend state，失败或 + timeout 必须保留 state 并传播。 +- 禁止：以单一 frontend flag 代替 AR scheduler 控制；在 sleep RPC 后才阻止 `generate()`；让 + wake 隐式 resume；因已 paused 跳过不同 stage scope 或 cache reset；将任意同名 `*_async` helper + 绕过 collective timeout；在 abort ack 前 pop request state，或将 abort failure 当成功。 +- 验收：覆盖 AR-only、diffusion-only 和 mixed stage 路由；sleep 进行时 generation 等待，AR + sleep → wake 后仍需显式 resume，而 diffusion-only sleep → wake 可恢复 admission；重复/定向 + pause 仍调用 scheduler 与 cache clear，非法 stage ID 产生明确错误，fast path 仅命中四个方法且 + timeout 生效；abort success 后才清理 frontend state，orchestrator error 与 timeout 时 state 保留。 + ^[PR #6084] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index e7e662450..cd7b92e9f 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -23,7 +23,7 @@ confidence: high | batched chat、fan-out/fan-in、sub-request ID、choice collapse、whole-batch error | `batch-chat-contract`：`SERV-4i`–`4k` | `api_server.py::create_batch_chat_completion` → `batch_serving.py::OmniOpenAIServingChatBatch` → ordinary chat completion children | | `chat_template_kwargs`、raw HTTP/SDK `extra_body`、text/audio modalities、choices、空音频 | `chat-multimodal-contract`：`SERV-4c` + 命中模型规则 | upstream `ChatCompletionRequest` → `serving_chat.py::{OmniOpenAIServingChat._preprocess_chat,OmniOpenAIServingChat.chat_completion_full_generator,OmniOpenAIServingChat._create_text_choice,OmniOpenAIServingChat._create_audio_choice}` | | endpoint restriction、route/app-state guard、capability、公开 400 | `endpoint-capability`：`SERV-4c`, `SERV-4d`, `SERV-5d` | endpoint policy → `api_server.py::build_app` assembled app → public handler | -| sleep/wake、partial stage/tag、idempotency、ACK、generation admission | `engine-lifecycle`：`SERV-5a`, `SERV-5b` | `entrypoints/async_omni.py::{AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni.generate}` → `worker/base.py::{handle_sleep_task,handle_wake_task}` / `diffusion/worker/diffusion_worker.py` | +| pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup acknowledgment | `engine-lifecycle`：`SERV-5a`, `SERV-5b`, `SERV-5m` | `entrypoints/async_omni.py::{AsyncOmni.pause_generation,AsyncOmni.resume_generation,AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni._abort}` → `engine/{stage_pool,orchestrator,async_omni_engine}.py` → AR EngineCore helper / diffusion worker RPC | | stage/replica death、`EngineDeadError`、`StageUnavailableError`、health/readiness、eviction | `engine-lifecycle`：[SERV-5f](rules-fault-isolation.md#serv-5f-stage-death-按-replica-隔离request-fatal-不等于-process-fatal) | `engine/stage_pool.py` live set → `engine/orchestrator.py` poll/dispatch/cleanup → `entrypoints/{omni_base,async_omni}.py` error transport | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | @@ -43,7 +43,7 @@ confidence: high | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | -| `engine-lifecycle` | sleep/wake、partial stage/tag、ACK、generation admission、factory 状态矩阵、TTS adapter detection、replica fault isolation | `SERV-5a`–`SERV-5e`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | +| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、factory 状态矩阵、TTS adapter detection、replica fault isolation | `SERV-5a`–`SERV-5e` 与 `SERV-5m`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints | `SERV-4a`–`4h`, `SERV-4l`–`4o`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4f79c329a..660854ba7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 072bfc02`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d15b7ca3`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -91,7 +91,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to 072bfc02 \ + --to d15b7ca3 \ --repo \ --mode report-only ``` From 936fa7f2192bbf7644661c6dcaaf68a6c9e6f9eb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 16:31:09 +0800 Subject: [PATCH 277/465] knowledge(vllm-omni): sync f446468c (PR #6535) Upstream-Commit: f446468c03e96a2359b74e372c258eb83e04d11a Upstream-PR: #6535 Knowledge-Delta: advance audited source pins for the WeChat documentation asset refresh --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- knowledge/repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4d26c9b64..99e7a4f9e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 072bfc02dd74cb0eb5c2f2a914e5dbbddba43b65 + previous_audited_sha: d15b7ca3e12ce969a37547f7480470b5b755dbf3 audited_ref: main - audited_sha: d15b7ca3e12ce969a37547f7480470b5b755dbf3 + audited_sha: f446468c03e96a2359b74e372c258eb83e04d11a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6b24bcc5e..a11d6ed60 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d15b7ca3e12ce969a37547f7480470b5b755dbf3`. The canonical machine baseline +f446468c03e96a2359b74e372c258eb83e04d11a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f30583b7c..9a0c7c46d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d15b7ca3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f446468c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2957e8a4c..db0ad3c33 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d15b7ca3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f446468c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 41c167cd1..d98960274 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d15b7ca3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f446468c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7f275c4b9..cfefbec4e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d15b7ca3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f446468c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 047d3c739..15e528622 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d15b7ca3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f446468c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a7518f58f..87ec6f42d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d15b7ca3` 验证存在 +- 源码校验：以上路径均已在 `main @ f446468c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 660854ba7..16905ecde 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d15b7ca3`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f446468c`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -91,7 +91,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to d15b7ca3 \ + --to f446468c \ --repo \ --mode report-only ``` From b035c08032713d98494dcb701e6bed993aaa9fd2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 16:36:37 +0800 Subject: [PATCH 278/465] knowledge(vllm-omni): sync 0e4b28f4 (PR #6557) Upstream-Commit: 0e4b28f491721af618525f4ba9f679f0eb5a7157 Upstream-PR: #6557 Knowledge-Delta: advance audited pins for the Wan DMD alignment alias exclusion --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- knowledge/repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 99e7a4f9e..2bc747bb9 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d15b7ca3e12ce969a37547f7480470b5b755dbf3 + previous_audited_sha: f446468c03e96a2359b74e372c258eb83e04d11a audited_ref: main - audited_sha: f446468c03e96a2359b74e372c258eb83e04d11a + audited_sha: 0e4b28f491721af618525f4ba9f679f0eb5a7157 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a11d6ed60..d94b9df1f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f446468c03e96a2359b74e372c258eb83e04d11a`. The canonical machine baseline +0e4b28f491721af618525f4ba9f679f0eb5a7157`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9a0c7c46d..ea19c54ec 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f446468c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0e4b28f4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index db0ad3c33..d5565b501 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f446468c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 0e4b28f4` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d98960274..0f4458751 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f446468c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0e4b28f4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cfefbec4e..f6e8807aa 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f446468c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0e4b28f4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 15e528622..7a942c33e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f446468c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0e4b28f4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 87ec6f42d..1eb9c74d2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f446468c` 验证存在 +- 源码校验：以上路径均已在 `main @ 0e4b28f4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 16905ecde..066ddf463 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f446468c`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0e4b28f4`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -91,7 +91,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to f446468c \ + --to 0e4b28f4 \ --repo \ --mode report-only ``` From d0ba4bbc17d42026a215250a5ad014091567f3c6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 16:42:31 +0800 Subject: [PATCH 279/465] knowledge(vllm-omni): sync aa69acbd (PR #6234) Upstream-Commit: aa69acbd090d4b6e021ed2bc22966fca03504572 Upstream-PR: #6234 Knowledge-Delta: require authoritative Diffusers shard manifests and fail-closed unindexed loading --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-checkpoint-loading.md | 17 ++++++++++++++++- .../vllm-omni/components/diffusion/rules.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 11 files changed, 28 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2bc747bb9..4c858333a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f446468c03e96a2359b74e372c258eb83e04d11a + previous_audited_sha: 0e4b28f491721af618525f4ba9f679f0eb5a7157 audited_ref: main - audited_sha: 0e4b28f491721af618525f4ba9f679f0eb5a7157 + audited_sha: aa69acbd090d4b6e021ed2bc22966fca03504572 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d94b9df1f..a40e93874 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0e4b28f491721af618525f4ba9f679f0eb5a7157`. The canonical machine baseline +aa69acbd090d4b6e021ed2bc22966fca03504572`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ea19c54ec..cfaed15c9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0e4b28f4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ aa69acbd` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d5565b501..19612429f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0e4b28f4` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ aa69acbd` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index 88b03a14e..1518a249c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6279", "PR #6445", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, "PR #5531", "PR #4061"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6234", "PR #6279", "PR #6445", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, "PR #5531", "PR #4061"] confidence: high --- @@ -154,3 +154,18 @@ confidence: high - 禁止：用运行时 YAML parser、原始 NVLabs key remap、layout sniff 或宽泛下载模式弥补不兼容的 checkpoint；把命名为 `SanaWmTwoStagesPipeline` 的旧两阶段 repo 当作当前 Stage-1 registry 自动发现成功，或静默跳过缺文件、未知/重复/shape 不符权重。 - 验收：local path 与 HF snapshot 都验证上述文件、`model_index.json` class 和 transformer config 读取；weight loader 对 unknown/duplicate/shape mismatch fail fast 并由 strict coverage 检查 expected keys；Stage-1 smoke 证明不下载/初始化 refiner，旧 layout 只能在显式且兼容的 class override 下运行。^[PR #4061] +## DIFF-2y — Diffusers component index 必须是唯一的 shard manifest + +- 触发：修改 Diffusers component 的 safetensors index resolution、Hub/cache download、subfolder + path、`allow_patterns` 或本地 checkpoint discovery。 +- 强制：没有显式 `allow_patterns_overrides` 时，先在 local path 或配置的 Hub cache/remote + 解析 component-local index；保留 `subfolder`、`revision`、`cache_dir` 和 offline + `local_files_only` 语义。唯一 index 的非空 `weight_map` 是唯一权威 shard manifest：只下载并加载 + 去重、稳定排序后的其中文件，且任何缺失 shard 都失败。没有 index 时，必须拒绝同一 shard + family 出现冲突的 `of-N` totals。 +- 禁止：因离线模式跳过已缓存 index、重复拼接 subfolder、用宽泛 glob 混入 snapshot 的陈旧 + shard，或让目录枚举顺序决定 loader 行为。显式 override 是最高优先级选择，不能被 index + 悄然改写。 +- 验收：覆盖 local、online 与 offline-cache index resolution，断言 revision/cache/subfolder + 透传、只选择 manifest files 且缺文件失败；再覆盖显式 override、无 index 的冲突 shard + totals 拒绝，以及顺序无关的 manifest selection。^[PR #6234] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index f3aa38c61..182b23c30 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -38,7 +38,7 @@ confidence: high |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1j` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o` 见 [LoRA](rules-lora.md) | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s`, `2y` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r`, `2w` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o`, `2x` 见 [LoRA](rules-lora.md) | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j`（`4a`–`4i` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)） | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0f4458751..06b58401a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0e4b28f4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ aa69acbd` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f6e8807aa..379566abc 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0e4b28f4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ aa69acbd` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7a942c33e..63dbae0d8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0e4b28f4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ aa69acbd` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1eb9c74d2..f3515916b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0e4b28f4` 验证存在 +- 源码校验：以上路径均已在 `main @ aa69acbd` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 066ddf463..8565f7aab 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0e4b28f4`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ aa69acbd`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -91,7 +91,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to 0e4b28f4 \ + --to aa69acbd \ --repo \ --mode report-only ``` From 94dcc8e324913667bbecf5fce6fa9a9068028071 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 16:53:18 +0800 Subject: [PATCH 280/465] knowledge(vllm-omni): sync d150a4fd (PR #5810) Upstream-Commit: d150a4fde77d15d466102323a4048b0a8631d74c Upstream-PR: #5810 Knowledge-Delta: capture H3 step execution, packed-request isolation, and distributed failure gaps --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/step-and-batching.md | 13 ++++++++++++- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../repos/vllm-omni/models/minimax-h3/_index.md | 1 + .../vllm-omni/models/minimax-h3/rules-cache-task.md | 11 +++++++++-- .../repos/vllm-omni/models/minimax-h3/rules.md | 1 + 13 files changed, 34 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4c858333a..53bdd471a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0e4b28f491721af618525f4ba9f679f0eb5a7157 + previous_audited_sha: aa69acbd090d4b6e021ed2bc22966fca03504572 audited_ref: main - audited_sha: aa69acbd090d4b6e021ed2bc22966fca03504572 + audited_sha: d150a4fde77d15d466102323a4048b0a8631d74c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a40e93874..91cc0f01a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -aa69acbd090d4b6e021ed2bc22966fca03504572`. The canonical machine baseline +d150a4fde77d15d466102323a4048b0a8631d74c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index cfaed15c9..a1138ec64 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ aa69acbd` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d150a4fd` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 19612429f..f60feb5db 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ aa69acbd` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d150a4fd` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md index 381516d02..85b233380 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md +++ b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-02 type: guide tags: [vllm-omni, components, diffusion] -sources: ["PR #5599", "PR #6045", docs/design/feature/diffusion_continuous_batching.md, docs/user_guide/diffusion/execution_modes.md, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/sched/step_scheduler.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_step_pipeline.py] +sources: ["PR #5599", "PR #5810", "PR #6045", docs/design/feature/diffusion_continuous_batching.md, docs/user_guide/diffusion/execution_modes.md, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/sched/step_scheduler.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_step_pipeline.py] --- # Step 执行合同与 batching 模式 @@ -62,6 +62,17 @@ step mode 当前拒绝所有 diffusion cache backend，但这不等于拒绝 int runner 对新 admission 先调用 `receive_multi_kv_cache_distributed()`，再 `prepare_encode()`，且有 顺序测试。限制描述必须区分这两个机制。 +## 多 document packing 是模型能力，不是 continuous batching 的默认语义 + +通用 step contract 只要求 scheduler 能交付一组 state；是否能把它们塞进**一次** DiT forward， +取决于所有 resolved attention consumer 是否实际支持并消费 multi-document `cu_seqlens`。不支持、 +Ring attention 或 task-specific DiT 不同都应退回逐 request forward，且不可通过 backend 名称或 +`max_num_seqs` 推断安全。MiniMax-H3 的 64-row tail、video/audio row split、Cache-DiT/DLO/multi-output +限制和 rank-0 prepare broadcast 是其模型 owner 合同，不应泛化给其他 pipeline。target runner 的 +`_dit_any_rank_failed()` 因缺失 DiT-group accessor 而只返回 local failure，且未按 group backend +选择 signal device；在补齐并以多 rank 负向用例证明前，通用 runner 不提供 failure synchronization +保证。^[PR #5810] + ## output stream 是统一生命周期 streaming 与 non-streaming caller 都消费 `DiffusionEngine.step_streaming()` 对应的 per-request diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 06b58401a..90e81a9d3 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ aa69acbd` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d150a4fd` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 379566abc..29daf9f4c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ aa69acbd` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d150a4fd` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 63dbae0d8..2f031817e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ aa69acbd` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d150a4fd` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f3515916b..adae34dc0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ aa69acbd` 验证存在 +- 源码校验：以上路径均已在 `main @ d150a4fd` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8565f7aab..358f0ee30 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ aa69acbd`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d150a4fd`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -91,7 +91,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to aa69acbd \ + --to d150a4fd \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 3b95990b2..afa91dff6 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -72,6 +72,7 @@ confidence: high 交互与单 H100 校准边界见 MMH3-2g。 - distilled checkpoint 的 continuous RF `base_schedule`、四步语义与 partition 隔离见 MMH3-2h；它不替换 H3 的自定义 solver。 +- step execution 是 H3 的模型专有 stateful path：只在 resolved attention backend 能隔离多个 packed documents 时合并 request；否则仍逐 request 前向。request state、rank-0 preparation、Cache-DiT/DLO/multi-output 排除与有界验证证据见 MMH3-2k，不能由 generic runner guard 推断 multi-rank failure synchronization。^[PR #5810] - LightX2V Turbo v1.0 仅支持 legacy dynamic LoRA；精确 artifact mapping、packed QKV/FC1 binding、active-request sampling 合同与 offload 排除项见 MMH3-2j；转换后的 mixed-rank native PEFT checkpoint 不在该支持声明内。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md index fe9506833..5369f6831 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md @@ -4,13 +4,13 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5720", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py] +sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py] confidence: high --- # MiniMax H3 缓存与任务生命周期规则 -`MMH3-2c`–`MMH3-2j`：conditioned VAE 的确定性、modular task 选择，以及 request 级 +`MMH3-2c`–`MMH3-2k`：conditioned VAE 的确定性、modular task 选择，以及 request 级 Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期。触发信号见 [MiniMax H3 规则](rules.md) 的 Direct 代码快速入口；加载合同见 [加载规则](rules-loading.md)，媒体输入见 [媒体规则](rules-media.md)。 @@ -162,6 +162,13 @@ Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期 - 禁止：将 exact Turbo artifact 的损坏 metadata 静默交给 generic fallback；将任意 H3 PEFT checkpoint、prefusion、multi-LoRA composition 或 Ref2VA 宣称为此功能支持；在 model-level CPU、layerwise 或 distributed-layerwise offload 下激活 Turbo，因为 legacy wrapper 的 LoRA tensors 不随 base parameter lifecycle 搬运。 - 验收：覆盖 file/metadata/alpha/target/pair/shape rejection、QKV and FC1 packing、scale-zero/generic same-ID/Ref2VA lifecycle、steps/shifts errors 与三种 offload rejection；真实 artifact evidence 必须另绑定 exact checkpoint、task、topology、sampling 和 video/audio output。转换后的 native mixed-rank PEFT artifact 不属于本 PR 的 supported contract；其 generic fallback rank/scale mismatch 是 post-merge follow-up，不可用本页的 Turbo success 证明安全。^[PR #6476] +## MMH3-2k — H3 step execution 必须保持请求状态、attention 文档和 rank-0 prepare 隔离 + +- 触发：MiniMax H3 实现或修改 `prepare_encode`、`denoise_step`、`step_scheduler`、`post_decode`、`batched_packing.py`、step-mode attention metadata，或改变与 DLO、Cache-DiT、TeaCache、task-specific DiT 的组合。 +- 强制：request 与 step path 复用 request-input、denoise-row prepare 与 unpack helper；每个 `StepRequestState` 保持一份 video rows、audio rows、audio sigma schedule、anchors、shape 和 output，只有同宽 video rows 交给 runner split。可融合时每个 request 向 `cu_seqlens` 贡献一个 real-row document 和一个独立的 64-row alignment-tail document，单 DiT forward 后按原 request rows 回拆；不同 task DiT、Ring 或任一 resolved backend 不支持 multi-document packed varlen 时逐 request forward。rank-0-only reference preparation 的异常必须在任何后续 broadcast 前由 H3-local helper 同步为同一异常。step mode 必须明确拒绝 `num_outputs_per_prompt>1`、distributed layerwise offload 和 `quality=high` request-scoped Cache-DiT；这些状态不能安全跨交错 step 或 packed forward 共享。 +- 禁止：因 backend 名称为 `FLASH_ATTN`、平台默认或容量大于 1 就假设可合批；在多 document `cu_seqlens` 不被 backend 消费时去掉 mask；将不同宽度的 audio rows 混入 video batch；让 request-scoped Cache-DiT hooks、DLO resident window 或多输出 latent 跨 state 复用。不得把 H3-local broadcast 夸大为通用 runner 防护：target 的 `_dit_any_rank_failed()` 无法取得 DiT group 而退回 local flag，且 group-device 选择仍有 backend mismatch 风险。 +- 验收：CPU contract 覆盖 request/step helper parity、two-request packed document boundaries、video/audio split/reassembly、mixed-DiT/Ring/unsupported-backend fallback、abort-after-inflight-step，及三个明确 rejection；TeaCache extractor 继续以 omitted `num_requests=1` 走 request-mode path。真实权重只可在固定 checkpoint、seed、task、backend、topology 下比较 single vs packed outputs；并发 step run 的 packing composition 会随 arrival timing 改变，PR 的 4×H100 结果已显示 step rerun 和 request-vs-step 都非 bitwise 等价。有限 CPU、H100/B300 observations 和 workload-specific throughput 都不是所有 backend、small request、确定性或 multi-rank failure synchronization 的证明。^[PR #5810] + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 69929194f..98c52e887 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -33,6 +33,7 @@ confidence: high | TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | [MMH3-2g](rules-cache-task.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state) | custom enabler → H3 extractor → module-resident hook state / per-generation reset | | distilled、DMD2、`base_schedule`、4-step、sigma boundary | [MMH3-2h](rules-cache-task.md#mmh3-2h-distilled-sigma-schedule-按-partition-所有且以区间计步) | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | | LightX2V Turbo、legacy dynamic LoRA、packed QKV/FC1、five sigma points | [MMH3-2j](rules-cache-task.md#mmh3-2j-turbo-lora-只接受精确发布-artifact并与-h3-tasksampling-和-offload-lifecycle-共同门禁) | `lora.py::load_minimax_h3_turbo_lora` → pipeline hook/sampling validation → `DiffusionLoRAManager` | +| step execution、continuous batching、multi-document packing、rank-0 prepare failure | [MMH3-2k](rules-cache-task.md#mmh3-2k-h3-step-execution-必须保持请求状态attention-文档和-rank-0-prepare-隔离) | `pipeline_minimax_h3.py::{prepare_encode,denoise_step,step_scheduler}` → `batched_packing.py` → runner | | DLO、TP-local、resident layers、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | From b4b2a30e25c5dad2adf83ce0689241f06b7b054b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 16:58:21 +0800 Subject: [PATCH 281/465] knowledge(vllm-omni): sync a94b2580 (PR #6574) Upstream-Commit: a94b2580a193c8f0cd6d322bd69335f55f9d0401 Upstream-PR: #6574 Knowledge-Delta: advance audited pins for the Wan DMD alignment revert --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- knowledge/repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 53bdd471a..f89450a04 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: aa69acbd090d4b6e021ed2bc22966fca03504572 + previous_audited_sha: d150a4fde77d15d466102323a4048b0a8631d74c audited_ref: main - audited_sha: d150a4fde77d15d466102323a4048b0a8631d74c + audited_sha: a94b2580a193c8f0cd6d322bd69335f55f9d0401 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 91cc0f01a..f49d8bcea 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d150a4fde77d15d466102323a4048b0a8631d74c`. The canonical machine baseline +a94b2580a193c8f0cd6d322bd69335f55f9d0401`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a1138ec64..110edc65c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d150a4fd` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a94b2580` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f60feb5db..76ce38cce 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d150a4fd` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ a94b2580` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 90e81a9d3..ea7806ec3 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d150a4fd` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a94b2580` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 29daf9f4c..a8ebd81d1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d150a4fd` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a94b2580` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2f031817e..f8795ad4b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d150a4fd` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a94b2580` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index adae34dc0..c3dc8aefa 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d150a4fd` 验证存在 +- 源码校验：以上路径均已在 `main @ a94b2580` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 358f0ee30..8a70dc00f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d150a4fd`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a94b2580`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -91,7 +91,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 39c16d75 \ - --to d150a4fd \ + --to a94b2580 \ --repo \ --mode report-only ``` From 8be011a314aef22ca5a68d4bb2d2d5777ed050fb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:02:27 +0800 Subject: [PATCH 282/465] knowledge(vllm-omni): sync 678aa258 (PR #6578) Upstream-Commit: 678aa25864334286d778fb3853b3c34f4b2ef0a6 Upstream-PR: #6578 Knowledge-Delta: advance audited pins for restoration of the Wan DMD alignment exclusion --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- knowledge/repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f89450a04..60e9811c7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d150a4fde77d15d466102323a4048b0a8631d74c + previous_audited_sha: a94b2580a193c8f0cd6d322bd69335f55f9d0401 audited_ref: main - audited_sha: a94b2580a193c8f0cd6d322bd69335f55f9d0401 + audited_sha: 678aa25864334286d778fb3853b3c34f4b2ef0a6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f49d8bcea..20b6c6a30 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -a94b2580a193c8f0cd6d322bd69335f55f9d0401`. The canonical machine baseline +678aa25864334286d778fb3853b3c34f4b2ef0a6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 110edc65c..30073676f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a94b2580` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 678aa258` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 76ce38cce..e7ab7af92 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a94b2580` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 678aa258` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ea7806ec3..4c96021ab 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a94b2580` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 678aa258` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a8ebd81d1..4d4283d80 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a94b2580` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 678aa258` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f8795ad4b..8dddb1bbb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a94b2580` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 678aa258` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c3dc8aefa..0198332af 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a94b2580` 验证存在 +- 源码校验：以上路径均已在 `main @ 678aa258` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8a70dc00f..67fdc2186 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a94b2580`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 678aa258`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From d2052aa112469189526bf69b68ec6b6b21a63b16 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:07:43 +0800 Subject: [PATCH 283/465] knowledge(vllm-omni): sync 8fdb719a (PR #6581) Upstream-Commit: 8fdb719a415f0e618fc7ac4e088de7947009e6e5 Upstream-PR: #6581 Knowledge-Delta: strengthen the explicit post-wake AR admission-resume contract with E2E evidence --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules-engine-lifecycle.md | 7 ++++--- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 10 files changed, 14 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 60e9811c7..35a1e4708 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a94b2580a193c8f0cd6d322bd69335f55f9d0401 + previous_audited_sha: 678aa25864334286d778fb3853b3c34f4b2ef0a6 audited_ref: main - audited_sha: 678aa25864334286d778fb3853b3c34f4b2ef0a6 + audited_sha: 8fdb719a415f0e618fc7ac4e088de7947009e6e5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 20b6c6a30..edcd41aca 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -678aa25864334286d778fb3853b3c34f4b2ef0a6`. The canonical machine baseline +8fdb719a415f0e618fc7ac4e088de7947009e6e5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 30073676f..d94b484e4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 678aa258` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8fdb719a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e7ab7af92..b36c1d596 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 678aa258` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8fdb719a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4c96021ab..ee4d143eb 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 678aa258` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8fdb719a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4d4283d80..7b076dd34 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 678aa258` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8fdb719a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8dddb1bbb..3763ffdec 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 678aa258` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8fdb719a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0198332af..579451aeb 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 678aa258` 验证存在 +- 源码校验：以上路径均已在 `main @ 8fdb719a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 29dca0b3d..b785a77ee 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346"] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py] confidence: high --- @@ -151,10 +151,11 @@ confidence: high wake 隐式 resume；因已 paused 跳过不同 stage scope 或 cache reset；将任意同名 `*_async` helper 绕过 collective timeout；在 abort ack 前 pop request state，或将 abort failure 当成功。 - 验收：覆盖 AR-only、diffusion-only 和 mixed stage 路由；sleep 进行时 generation 等待，AR - sleep → wake 后仍需显式 resume，而 diffusion-only sleep → wake 可恢复 admission；重复/定向 + sleep → wake 后仍需显式 resume，跨 AR+diffusion 的 sleep/wake E2E 必须在 post-wake + `generate()` 前显式调用 `resume_generation()`，而 diffusion-only sleep → wake 可恢复 admission；重复/定向 pause 仍调用 scheduler 与 cache clear，非法 stage ID 产生明确错误，fast path 仅命中四个方法且 timeout 生效；abort success 后才清理 frontend state，orchestrator error 与 timeout 时 state 保留。 - ^[PR #6084] + ^[PR #6084] ^[PR #6581] ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 67fdc2186..10a89a35a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 678aa258`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8fdb719a`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 90099a87c970f60fdf47f66e50a6907940406914 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:16:23 +0800 Subject: [PATCH 284/465] knowledge(vllm-omni): sync 8360346a (PR #6486) Upstream-Commit: 8360346a6637bf9cc6ce4414549f7c422722841e Upstream-PR: #6486 Knowledge-Delta: capture final-layout HWR integration with no-AllGather DLO and transactional lease handoff --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 6 +++--- .../diffusion/rules-checkpoint-loading.md | 19 ++++++++++++++++++- .../diffusion/rules-component-lifecycle.md | 16 +++++++++++++++- .../vllm-omni/components/diffusion/rules.md | 5 +++-- .../components/distributed/_index.md | 2 +- .../components/host-weight-runtime/_index.md | 4 ++-- .../components/host-weight-runtime/rules.md | 16 +++++++++++++++- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../models/minimax-h3/rules-loading.md | 6 +++++- 15 files changed, 70 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 35a1e4708..1b7c3f745 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 678aa25864334286d778fb3853b3c34f4b2ef0a6 + previous_audited_sha: 8fdb719a415f0e618fc7ac4e088de7947009e6e5 audited_ref: main - audited_sha: 8fdb719a415f0e618fc7ac4e088de7947009e6e5 + audited_sha: 8360346a6637bf9cc6ce4414549f7c422722841e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index edcd41aca..84b83f594 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8fdb719a415f0e618fc7ac4e088de7947009e6e5`. The canonical machine baseline +8360346a6637bf9cc6ce4414549f7c422722841e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d94b484e4..d21fcb516 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8fdb719a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8360346a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b36c1d596..4cbe75a3b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8fdb719a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8360346a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -34,7 +34,7 @@ sources: [] | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | | Wan VAE height/width spatial reshard、empty tail、attention extent | [Wan spatial-shard rules](rules-wan-spatial-shard.md) | -| multi-DiT、dotted `_dit_modules` 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | +| multi-DiT、dotted `_dit_modules`、loader-to-offloader handoff 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | | local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | @@ -47,4 +47,4 @@ sources: [] | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | | 实验性 world-model session 生命周期、LRU 与内存统计边界 | [session state](session-state.md) | -| checkpoint remap、HSDP/FSDP、component quantization 与在线量化加载 | [checkpoint 与加载合同](rules-checkpoint-loading.md) | +| checkpoint remap、HSDP/FSDP、final-layout HWR、component quantization 与在线量化加载 | [checkpoint 与加载合同](rules-checkpoint-loading.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index 1518a249c..fc69573de 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6234", "PR #6279", "PR #6445", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, "PR #5531", "PR #4061"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6234", "PR #6279", "PR #6445", "PR #6486", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, "PR #5531", "PR #4061"] confidence: high --- @@ -169,3 +169,20 @@ confidence: high - 验收：覆盖 local、online 与 offline-cache index resolution，断言 revision/cache/subfolder 透传、只选择 manifest files 且缺文件失败；再覆盖显式 override、无 index 的冲突 shard totals 拒绝，以及顺序无关的 manifest selection。^[PR #6234] + +## DIFF-2z — final-layout HWR 只能接入 eligible no-AllGather DLO + +- 触发：修改 Diffusers loader 的 host-weight runtime selection、final-layout producer/restorer、 + `HostWeightPlan`，或 DLO loader/backend handoff。 +- 强制：在 import、source preparation、identity/store construction 和 observer event 之前完成 + mode/DLO/no-AllGather eligibility gate；disabled、DLO-disabled 和 AllGather 保持既有 + checkpoint-mmap 或 ordinary-loader 路径且零 HWR interaction。eligible warm hit 必须跳过 + ordinary DiT materialization，plan/commit 后把 exact final-layout tensors 作为 rank-local DLO + source；preferred miss 才 canonical load 并 post-load publish，required 对 miss 或 + incompatible artifact fail startup，不能借 producer bootstrap。 +- 禁止：把 preferred 当成对 identity/configuration/compatibility error 的宽泛 fallback；让 + HWR warm restore 重走 ordinary DiT loader、checkpoint mmap 或 byte-changing finalization；把 + no-AllGather 的 rank-local staging 扩展成跨 rank collective 或 generic HWR support。 +- 验收：覆盖所有 zero-interaction gates、preferred hit/miss、required miss、mixed/dedicated + source rejection、warm path zero ordinary DiT materialization，以及 shared finalization 前后 + restored tensor byte/backing-pointer equality；验证 checkpoint mmap control path 不变。^[PR #6486] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md index f64dff8fe..534a96ad2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5720", "PR #5853", "PR #5884", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py", "PR #6072"] +sources: ["PR #5720", "PR #5853", "PR #5884", "PR #6486", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/offloader/startup.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py", "PR #6072"] confidence: high --- @@ -94,3 +94,17 @@ residency 留在模型 owner。规则入口与其他共享机制仍见 [Diffusio - 禁止：以可调用属性探测替代 protocol、只依赖 generic forward hook 管理 direct VAE call、在没有 activation/release scope 时直接调用 hooked component，或在启停失败后遗留部分 hooks 与设备状态。 - 验收：CPU/mock 覆盖 protocol pipeline 与普通 pipeline、真实 discovery 的 DiT/stage 列表、初始 DiT CPU residency、direct component activation failure 的 finally cleanup，以及 enable/disable 后所有 hook 清理；真实 accelerator 的性能与完整模型显存结论仍需独立验证。^[PR #6072] +### DIFF-12a — loader-to-offloader startup handoff 必须单次消费并事务性回收 + +- 触发：loader 产生 backend-owned state、runner/offloader startup boundary，或 backend enable/ + initial-prefetch failure cleanup。 +- 强制：loader 把 process-local startup state 附在 loaded pipeline 上，runner 只调用 generic + enable boundary；该 boundary 必须 take-and-remove state exactly once。backend enable 从 hook/ + staging/lease acquisition 到 initial prefetch 失败时先 quiesce and disable，再关闭仍属 loader 的 + state；已提交 final-layout restore 在 preferred mode 只能以 fresh canonical model retry，required + mode 必须传播失败。 +- 禁止：让 runner 解释 HWR policy 或重复 transfer plan；复用 commit 后的 disposable model；在 + async work、partial hooks 或 staging 仍存活时释放 lease，或让 retry 再次 lookup/publish/mmap。 +- 验收：覆盖 absent/taken startup state、backend construction/enable/prefetch failure、single fresh + retry 与 required no-retry；断言 partial backend disable、carrier/lease close、fresh path bypasses + HWR and checkpoint mmap。^[PR #6486] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 182b23c30..abc787855 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -23,12 +23,13 @@ confidence: high | CUDA Graph、compile、fused solver/norm/RoPE、FA determinism、eager parity、tensor dtype/device、async output/teardown | `execution-parity`：`DIFF-1a`–`1j` | `compile.py::regionally_compile` → shared layer/backend → model denoise/output pump/shutdown | | seed、request-local generator、guidance=0、并发 RNG、batched generators | `execution-parity`：`DIFF-1b` | `inputs/data.py::OmniDiffusionSamplingParams` → runner `_initialize_generator` → request-batch generator collate | | ModelOpt/checkpoint adapter、weight/scale remap、unknown tensor、resolution path | `checkpoint-distributed`：`DIFF-2a` | `diffusers_loader.py::{_get_checkpoint_adapter,load_weights}` → `modelopt.py::{_resolve_target_and_output_names,adapt}` | -| host-weight artifact、source identity、layout/dtype、warm restore | `checkpoint-distributed`：`DIFF-2d` | `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | +| host-weight artifact、source identity、layout/dtype、warm restore、no-AllGather DLO selection | `checkpoint-distributed`：`DIFF-2d`, `DIFF-2z` | `model_loader/host_weight_loader.py::_resolve_hwr` → `model_loader/host_weights/{source_identity,contracts,identity_adapter}.py` → policy/restorer | | HSDP/FSDP、`fully_shard`、DeviceMesh、packed/scalar parameter、FP8 | `checkpoint-distributed`：`DIFF-2b` | `distributed/hsdp.py::{apply_hsdp_to_model,shard_model}` → loader `_load_model_with_hsdp` → `hsdp_fp8.py` | | distributed layerwise offload、AllGather、异构 block、shared buffer | `checkpoint-distributed`：`DIFF-2e` | `offloader/distributed_layerwise_backend.py::{DistributedLayerwiseOffloadHook.initialize_hook,prefetch_layer,DistributedLayerwiseOffloadBackend._allocate_shared_buffers}` | | DLO+AllGather、DP wave、result queue、shutdown、constructor cleanup | `checkpoint-distributed`/`execution-parity`：`DIFF-2k`, `DIFF-1d/1g/1h/1j` | request compatibility key → multi-rank RPC → worker-owned queue/pump → bounded teardown | | component quantization、text encoder/transformer/VAE 独立配置、owner prefix、meta/offload | `checkpoint-distributed`：`DIFF-2c` | `quantization/factory.py::{build_quant_config,resolve_quant_config_from_disk}` → `component_config.py::ComponentQuantizationConfig.resolve` → `data.py::_propagate_quantization_from_tf_config` → component linear consumer | | modular/multi-DiT、`_dit_modules`、request-scoped Cache-DiT/compile/SP/LoRA/offload lifecycle | `checkpoint-distributed`：`DIFF-2f`–`2j` | request batch key → model policy/cache runtime；pipeline component list → shared consumers | +| loader-owned offload plan、backend enable/prefetch failure、fresh canonical retry | `checkpoint-distributed`：`DIFF-12a` | `offloader/startup.py` → `offloader/__init__.py::enable_offload_backend` → runner | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | Wan VAE spatial shard、gather/trim/reshard、empty tail、attention extent | `distributed-vae`：`DIFF-3b` | `distributed/autoencoders/wan_spatial_shard.py` → patched decoder attention/conv → gather final frame | | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4j` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | @@ -38,7 +39,7 @@ confidence: high |---|---|---| | `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | | `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1j` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s`, `2y` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r`, `2w` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o`, `2x` 见 [LoRA](rules-lora.md) | +| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s`, `2y`, `2z` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r`, `2w`, `12a` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o`, `2x` 见 [LoRA](rules-lora.md) | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j`（`4a`–`4i` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)） | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ee4d143eb..9e460511a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8fdb719a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8360346a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md index 579b72660..b2fbb5deb 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: index tags: [vllm-omni, components] -sources: ["PR #6419", vllm_omni/host_weight_runtime/] +sources: ["PR #6419", "PR #6486", vllm_omni/host_weight_runtime/] confidence: high --- @@ -23,4 +23,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| typed failure、lease/store concurrency、restore transaction | [rules](rules.md) | +| typed failure、lease/store concurrency、single-take carrier、restore transaction | [rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md index 6a1b54ee7..8b345971f 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, components] -sources: ["PR #6419", "PR #6445", vllm_omni/host_weight_runtime/, "PR #6427"] +sources: ["PR #6419", "PR #6445", "PR #6486", vllm_omni/host_weight_runtime/, "PR #6427"] confidence: high --- @@ -58,3 +58,17 @@ confidence: high post-load policy 当 cache miss。 - 验收：plan failure 保持原 model，commit failure 后 fresh model fallback；schema/version mismatch、 malformed artifact 和 unsupported policy 均 fail-closed。 ^[PR #6419] ^[PR #6445] + +## HWR-2b — cross-boundary lease handoff 必须是 process-local single-take + +- 触发：consumer 把 restored lease 从 loader 交给 runner/backend，或修改 carrier serialization、 + startup cleanup、store publication hardening。 +- 强制：commit 后用 carrier 保留唯一 loader-owned open lease；transport owner 在任何 async work + 前 `take()` 一次，随后只由 backend teardown/abort close。carrier 未被 take 时的 close 释放 lease， + taken/closed carrier 拒绝再次 take，且不得可序列化。filesystem artifact 在 atomic rename 后再 + harden；harden failure quarantine artifact。 +- 禁止：复制 lease、让 loader and backend 同时 close，或在 source directory immutable 前 rename； + 不能把 overlayfs-compatible publication 顺序改回 rename 前只读目录。 +- 验收：验证 duplicate take、serialization、loader-side abort、backend-side teardown、enable failure + cleanup 与 closed lease rejection；filesystem test 覆盖 writable staging → atomic rename → read-only + artifact，以及 hardening failure quarantine。^[PR #6486] diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7b076dd34..d25f8c3c7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8fdb719a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8360346a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3763ffdec..691c94284 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8fdb719a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8360346a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 579451aeb..dffb8aa1d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8fdb719a` 验证存在 +- 源码校验：以上路径均已在 `main @ 8360346a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 10a89a35a..88e959215 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8fdb719a`（2026-08-24）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8360346a`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md index a784240da..21db7aa58 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-loading.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #5910", "PR #6213", "PR #6445"] +sources: ["PR #5706", "PR #5737", "PR #5824", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, "PR #5910", "PR #6213", "PR #6445", "PR #6486"] confidence: high --- @@ -68,3 +68,7 @@ load 完成时 dynamic quantize，text encoder、VAE 和非 eligible projection - 禁止：用 dtype 字符串代替版本化 model ABI；遗漏 persistent buffer 或模型要求保留的 FP32 parameter/buffer；把 contract 声明或 validator 存在误认为 loader 已启用、DLO 已接入或 artifact 已具备跨拓扑性能保证。 - 验收：contract test 断言 `vllm-omni.diffusion.final-layout-tensors-v1`、`minimax-h3-dit`、版本与 validator；reduced MiniMax H3 真实 ownership 同时覆盖 BF16 tensor、模型要求的 FP32 tensor 和 persistent buffer，并在 one-shot restore commit 后执行 validator；另以 exact identity、tensor coverage 与 source-change guard 覆盖失败时不变更模型。^[PR #6445] +H3 的该 contract 当前只为 BF16 no-AllGather DLO final-layout consumer 接通：artifact identity +区分 TP rank/size 与 SP layout，等价 DP replicas 才共享；TP/SP/layout/model revision 改变时必须以 +`preferred` 重新产生 artifact，`required` 只消费 exact hit。它不扩大 direct checkpoint mmap、 +online quantization 或 AllGather 的支持范围。^[PR #6486] From 1e443be26d3aef7ae1171971b5ea3f0d1e115a04 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:25:12 +0800 Subject: [PATCH 285/465] knowledge(vllm-omni): sync 4af0ddc1 (PR #6523) Upstream-Commit: 4af0ddc1c2cf007dbbe69d9abab9f415dde805e6 Upstream-PR: #6523 Knowledge-Delta: capture test-helper ownership, startup rollback, and ready-gate regression coverage boundaries --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 12 +++++++++--- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../repos/vllm-omni/models/qwen3-tts/rules.md | 15 +++++++++++++-- 11 files changed, 32 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1b7c3f745..db673ee51 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8fdb719a415f0e618fc7ac4e088de7947009e6e5 + previous_audited_sha: 8360346a6637bf9cc6ce4414549f7c422722841e audited_ref: main - audited_sha: 8360346a6637bf9cc6ce4414549f7c422722841e + audited_sha: 4af0ddc1c2cf007dbbe69d9abab9f415dde805e6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 84b83f594..fca7a9b89 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8360346a6637bf9cc6ce4414549f7c422722841e`. The canonical machine baseline +4af0ddc1c2cf007dbbe69d9abab9f415dde805e6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 56c5820a2..852bb5c1f 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/runtime.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -20,7 +20,7 @@ confidence: high | regression/guard、route census、middleware、mutation test | `OMNI-CI-1b` | 公开 app/handler → guard test；先证明旧实现会失败 | | `NIGHTLY`/`WEEKLY`/`NON_CRITICAL`、L1/L4/L5、coverage、scheduled bootstrap | `OMNI-CI-1c` | `upload_pipeline.py` → bootstrap YAML → ready/merge/nightly/weekly YAML → coverage helper | | pre-commit、SPDX、shellcheck、stability marker | `OMNI-CI-2a` | `.pre-commit-config.yaml`、`.buildkite/**`、`tools/**` | -| xdist、共享 worker、下载 fixture、进程池 | `OMNI-CI-2b` | `tests/conftest.py`、`tests/helpers/**`、`tests/model_tests/**` | +| xdist、共享 worker、helper 拆分/启动回滚、下载 fixture、进程池 | `OMNI-CI-2b` | `tests/conftest.py`、`tests/helpers/{client,clean,runtime,stage_config}.py`、`tests/model_tests/**` | | 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | | diffusion tiny builder、model settings、alignment exclusion、重模型 OOM | `OMNI-CI-2d` | `tests/model_tests/diffusion/{diff_model_builders,model_settings,test_alignment}.py` → common offline tests | | perf baseline、hardware label、DFX result artifact、assert-baseline | `OMNI-CI-3a` | `tests/dfx/conftest.py`、`tests/dfx/perf/scripts/run_benchmark.py`、`run_diffusion_benchmark.py`、`tests/dfx/perf/tests/**` | @@ -111,10 +111,16 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 - 触发：xdist、共享下载 fixture、长寿命 judge/transcriber worker、进程池 retry，或 spawned `torch.distributed` process group。 - 强制：任务级异常后丢弃污染 worker；submit/result 串行，只有进程崩溃可 retry-once；传给 xdist 的参数可序列化，依赖固定兼容 major；共享文件在读锁前显式检查存在，写入保持锁语义。spawned `torch.distributed` 必须为每个 group 传入新鲜 `file://` rendezvous URL，不能先释放 TCP port 再把其数字交给子进程；`get_open_port` 只用于真实 TCP service。 +- 共享 helper 的边界必须清晰：请求 client 从 `tests/helpers/client.py`、环境/设备清理由 + `clean.py`、server/runner 生命周期由 `runtime.py` 提供；pytest plugin `conftest.py` 只注册 + fixture，调用方不得依赖它的隐式或 star-import re-export。`__init__` 或 `__enter__` 的启动失败 + 不会调用 `__exit__`，所以必须立即回滚已启动的 stage 子进程、关闭已打开的日志 FD 并清除临时 + 日志；只有显式 `VLLM_OMNI_KEEP_LOG` 调试选择可保留日志文件。测试 deploy helper 只接受新 + schema 的顶层 `stages`；不得静默回退 `stage_args`，缺失时 consumer error 必须指出配置文件。 - 禁止：普通 task error 复用同一进程；为 pytest 生命周期不可达的竞态堆测试；把并行参数或 活对象隐式跨进程传递；把 file rendezvous 套用于 Mooncake/RDMA 等真实监听服务。 - 验收：OOM、普通异常和 BrokenProcessPool 分别验证隔离/重试；online/offline xdist 均通过， - 并发 cache miss 只产生一个完整文件；spawned Gloo/NCCL group 使用独立 file rendezvous，真实 TCP service 仍使用 port helper。 ^[PR #6208] ^[PR #6339] ^[PR #6468] + 并发 cache miss 只产生一个完整文件；spawned Gloo/NCCL group 使用独立 file rendezvous，真实 TCP service 仍使用 port helper。 ^[PR #6208] ^[PR #6339] ^[PR #6468] ^[PR #6523] ## OMNI-CI-2c — 昂贵 engine fixture 复用必须恢复状态并隔离拓扑 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d21fcb516..8310a9ce7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8360346a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4af0ddc1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 4cbe75a3b..5d21d891f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8360346a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4af0ddc1` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9e460511a..db65db659 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8360346a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4af0ddc1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index d25f8c3c7..220940bf7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8360346a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4af0ddc1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 691c94284..fecfa7c2b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8360346a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4af0ddc1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index dffb8aa1d..6be42f0a0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8360346a` 验证存在 +- 源码校验：以上路径均已在 `main @ 4af0ddc1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 88e959215..0143d8157 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8360346a`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4af0ddc1`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index 71d68297f..910b0214d 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] +sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6523", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] confidence: high --- @@ -17,7 +17,7 @@ confidence: high | PR 描述信号 | 规则组 | 第一批源码 | |---|---|---| | Qwen3-TTS、`qwen3_tts` pipeline | Q3TTS-1a/1b | `config/pipeline_registry.py::OMNI_PIPELINES["qwen3_tts"]`；`model_executor/models/qwen3_tts/pipeline.py` | -| `ref_audio`、x-vector、ICL、artifact-only reuse | Q3TTS-1a/1b | `entrypoints/openai/serving_speech.py::_qwen3_tts_can_use_ref_audio_artifact_only`、`_track_ref_audio_artifact_warmup`、`_mark_ref_audio_artifact_ready_for_request` | +| `ref_audio`、x-vector、ICL、artifact-only reuse | Q3TTS-1a/1b/1c | `entrypoints/openai/serving_speech.py::_qwen3_tts_can_use_ref_audio_artifact_only`、`_track_ref_audio_artifact_warmup`、`_mark_ref_audio_artifact_ready_for_request` | | talker/code2wav、delta frame、request cache、segmented graph | Q3TTS-3a/3b/3c/3d + Model Executor | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `qwen3_tts_code2wav.py::Qwen3TTSCode2Wav` → `segmented_graph_wrapper.py` | | OpenAI speech adapter | Q3TTS-1a/1b + Serving | `entrypoints/openai/tts_adapters/qwen3_tts.py::Qwen3TTSAdapter` → `serving_speech.py` | | NPU、RoPE、BNSD/BSND、`codec_chunk_ramp` | Q3TTS-2a | `platforms/npu/models/qwen3_tts_tokenizer_v2.py::_apply_rotary_pos_emb_npu` → `platforms/npu/layers/rotary_embedding.py::npu_rotary_mul_with_bsnd_fallback` | @@ -42,6 +42,17 @@ confidence: high - 验收：顺序测试证明 ICL → x-vector 只多一次重算、输出正确且后续同模式复用恢复； 若优化为 capability predicate，保留 x-vector artifact 不能服务 ICL 的单向边界。 ^[PR #5157] +## Q3TTS-1c — ref-audio artifact 回归必须保留 ready-gate 证明 + +- 触发：移动、合并或重标记同一 `ref_audio` 的 x-vector → ICL artifact regression test。 +- 强制：ready lane 必须仍选择 async-chunk server 上的该顺序；每步断言非空 audio，并在 ICL + 后再发一次 x-vector 请求以证明 engine 存活。full-model expansion 可以补充覆盖，不能替代 + `core_model`/`advanced_model` ready-gate case。 +- 禁止：只因测试正文仍在 expansion 文件，就声称 ready 已覆盖；被 marker selector 排除的 + `full_model` case 不构成 ready regression evidence。 +- 验收：ready 的实际 marker selector 收集并执行该 case；x-vector → ICL → x-vector 在同一 + audio 下通过，且在旧的 mode-agnostic artifact readiness 实现上失败。 ^[PR #5157] ^[PR #6523] + ## Q3TTS-2a — NPU RoPE 按真实 shape 在 BNSD 与 BSND fused path 间选择 - 触发：Qwen3-TTS 12 Hz tokenizer decoder 的 NPU RoPE，尤其启用短 From b68b5139025cdcede2c6cc8669c1b4ed15dc207c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:33:08 +0800 Subject: [PATCH 286/465] knowledge(vllm-omni): sync 50b8a150 (PR #5508) Upstream-Commit: 50b8a150f9df5ec17ff3a1a99ab113eb57d786d8 Upstream-PR: #5508 Knowledge-Delta: document native SANA-Video 2B T2V/I2V architecture, request contracts, and validation boundaries --- adapters/vllm_omni/release_baseline.yaml | 8 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 4 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 13 +++-- .../vllm-omni/models/sana-video/_index.md | 32 +++++++++--- .../models/sana-video/architecture.md | 51 +++++++++++++++++++ 11 files changed, 98 insertions(+), 22 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/sana-video/architecture.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index db673ee51..624c25bd5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8360346a6637bf9cc6ce4414549f7c422722841e + previous_audited_sha: 4af0ddc1c2cf007dbbe69d9abab9f415dde805e6 audited_ref: main - audited_sha: 4af0ddc1c2cf007dbbe69d9abab9f415dde805e6 + audited_sha: 50b8a150f9df5ec17ff3a1a99ab113eb57d786d8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 81 sha256: 9d5a685eae7ef099bb884557ca51b0e1b5f19c8555aa16bbd136c2081d95cd78 diffusion: - count: 65 - sha256: 5d50c8f9cc5aeb93ab1c6b9e2ba8eca145cb05f8343eef064e2a5ab873dfc556 + count: 67 + sha256: b11ab867faf8faa8f2b9a81254d178333abffd6e3adcf7c932399fabe162accb pipelines: count: 55 sha256: 9d5033c9a7ffd7ff636e32db0f9be2a549b7c16b034e13817e280f19a9e16769 diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fca7a9b89..0831ec867 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4af0ddc1c2cf007dbbe69d9abab9f415dde805e6`. The canonical machine baseline +50b8a150f9df5ec17ff3a1a99ab113eb57d786d8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8310a9ce7..afc3c198a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4af0ddc1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 50b8a150` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5d21d891f..fd3a59d1f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,8 +10,8 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4af0ddc1` 验证存在；本轮新增/扩展的 - async output、distributed layerwise offload 和 MiniMax H3 仍按各自模型/机制规则审查 +- 源码校验：以上子模块均已在 `main @ 50b8a150` 验证存在；本轮新增/扩展的 + async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index db65db659..29c55d672 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4af0ddc1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 50b8a150` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 220940bf7..05024dc1e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4af0ddc1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 50b8a150` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index fecfa7c2b..01da16cea 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4af0ddc1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 50b8a150` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6be42f0a0..86abd8977 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4af0ddc1` 验证存在 +- 源码校验：以上路径均已在 `main @ 50b8a150` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0143d8157..66993c4d6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4af0ddc1`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 50b8a150`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -36,7 +36,7 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | -| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 41 个模型族目录 | +| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 67 条 pipeline / 42 个模型族目录 | | Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 55 个 key | | Deploy YAML | `vllm_omni/deploy/*.yaml` | 84 份 | @@ -50,6 +50,11 @@ architecture、两个 pipeline key、两个 deploy YAML 及其 tokenizer/config/ PR #4820 将 `WanDMDPipeline` 注册为 Wan2.2 的同实现入口，因此 diffusion pipeline 计数回升至 65；它不是新的模型族或新的 `OMNI_PIPELINES` key。^[PR #4820] +SANA-Video 2B 新增 native `SanaVideoPipeline`（T2V）和 +`SanaImageToVideoPipeline`（I2V），二者共用 `sana_video` 模型族目录；它没有新增 +`OMNI_PIPELINES` key 或 deploy YAML。模型专有的 checkpoint、VAE 和 adapter 边界见 +[SANA Video](sana-video/_index.md)。^[PR #5508] + ## AR/omni 模型族（31） aura_omni、audex、bagel、cosyvoice3、covo_audio、dots_tts、dynin_omni、fish_speech、 @@ -58,13 +63,13 @@ mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、minimax_music3、moss_t moss_tts_nano、nemotron_voicechat、omnivoice、personaplex、qwen2_5_omni、qwen3_omni、 qwen3_tts、step_audio2、voxcpm2、voxtral_tts -## Diffusion 模型族（41） +## Diffusion 模型族（42） bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamid_omni、 dreamzero、ernie_image、flux、flux2、flux2_klein、glm_image、gr00t、helios、 hidream_image、hidream_o1_image、hunyuan_image3、hunyuan_video、internvla_a1、krea2、lance、 lingbot_video、lingbot_world、longcat_image、longcat_video、ltx2、magi_human、ming_flash_omni、 -minimax_h3、nextstep_1_1、omnigen2、omnivoice、ovis_image、pi0、qwen_image、sana_wm、sd3、 +minimax_h3、nextstep_1_1、omnigen2、omnivoice、ovis_image、pi0、qwen_image、sana_video、sana_wm、sd3、 sdxl、sensenova_u1、stable_audio、wan2_2、z_image ## OMNI_PIPELINES key（55） diff --git a/knowledge/repos/vllm-omni/models/sana-video/_index.md b/knowledge/repos/vllm-omni/models/sana-video/_index.md index 7f7dc9845..f8f53e8cb 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/_index.md +++ b/knowledge/repos/vllm-omni/models/sana-video/_index.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/sana_video/] +sources: ["PR #5508", vllm_omni/diffusion/models/sana_video/, vllm_omni/diffusion/registry.py, vllm_omni/model_extras/sana_video.py] confidence: high --- @@ -17,7 +17,7 @@ confidence: high ## 源码路径 -共 5 个文件： +共 5 个文件；native T2V/I2V pipeline、Transformer 与 output 都在： - `vllm_omni/diffusion/models/sana_video/` @@ -32,9 +32,29 @@ confidence: high - `vllm_omni/diffusion/worker/` → [diffusion](../../components/diffusion/_index.md) - `vllm_omni/diffusion/cache/` → [diffusion](../../components/diffusion/_index.md) -## checkpoint、尺寸与量化 - -- 上游没有 deploy 预设。模块也没有 docstring，因此 checkpoint、尺寸和量化差异在上游尚无声明，不在此处臆测；命中该 owner 的同步单元应补齐。 +## checkpoint、尺寸与兼容范围 + +- 支持 `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`（832×480）和 + `Efficient-Large-Model/SANA-Video_2B_720p_diffusers`（1280×704），均为单 GPU 的 + SANA-Video 2B T2V/I2V 范围；SANA-Video 2.0 5B/14B 不在此实现范围。 +- native pipeline 根据 checkpoint VAE class 选择共享分布式 Wan VAE 或 LTX-2 VAE wrapper； + 480p Wan VAE decode 使用 FP32，720p LTX-2 VAE 保持 pipeline dtype。scheduler 从 + checkpoint 加载 `DPMSolverMultistepScheduler`，不能以“native”推断零 Diffusers 依赖。 +- `--diffusion-load-format diffusers` 是兼容/reference backend。I2V 必须经 + `SanaImageToVideoPipeline` class hook 选择 Diffusers I2V pipeline；其 `frames` 参数由共享 + `num_frames` 映射，profile dummy run 仍允许一帧。 + +## 专有流程与验证 + +- T2V 用 native ReLU-kernel linear self-attention，prompt cross-attention 通过 vLLM-Omni + Attention；不要把 SANA-Video 2B 误归为 gated linear attention。I2V 编码输入图像、固定首个 + latent frame，并构造 first-frame conditioning mask。 +- 请求未指定视频帧数时，image-model 的 `num_frames=1` sentinel 在真实请求边界解析为 81； + dummy/profile run 保留一帧以避免昂贵启动路径。公开 extras 是 `clean_caption`、`motion_score` + 与 `use_resolution_binning`；native `clean_caption=True` 明确不支持。 +- 先运行 `tests/diffusion/models/sana_video/` 的组件回归；native/adapter、T2V/I2V、480p/720p + 的真实权重功能覆盖在 L4 serving expansion。golden 与 stage-alignment 是按需 correctness + 工具，不是默认 per-PR gate。 ## 什么时候查这里 @@ -50,4 +70,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | +| T2V/I2V、checkpoint、VAE、adapter 或验证边界 | [architecture](architecture.md) | diff --git a/knowledge/repos/vllm-omni/models/sana-video/architecture.md b/knowledge/repos/vllm-omni/models/sana-video/architecture.md new file mode 100644 index 000000000..6d1f5e81f --- /dev/null +++ b/knowledge/repos/vllm-omni/models/sana-video/architecture.md @@ -0,0 +1,51 @@ +--- +title: "SANA Video 架构" +created: 2026-09-04 +updated: 2026-09-04 +type: architecture +tags: [vllm-omni, models, diffusion] +sources: ["PR #5508", vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, vllm_omni/diffusion/request.py] +confidence: high +--- + +# SANA Video 架构 + +## 模型专有部分与共享模块的边界 + +`sana_video` owner 包含 native `SanaVideoPipeline`（T2V）、 +`SanaImageToVideoPipeline`（I2V）和 `SanaVideoTransformer3DModel`。Transformer 的 +self-attention 是模型专有 ReLU-kernel linear attention；prompt cross-attention 交给共享 +`vllm_omni.diffusion.attention.Attention`。VAE、component discovery、offload、request batch、 +postprocess 与 worker 生命周期仍属于共享 diffusion owner。 + +## 配置、checkpoint 与兼容范围 + +两个 released 2B checkpoint 由 `sample_size` 决定默认 profile：30 对应 480×832，22 对应 +704×1280。pipeline 只接受 Wan 或 LTX-2 video VAE class，分别接入对应 shared distributed +wrapper；scheduler 保留 checkpoint 的 `DPMSolverMultistepScheduler` 配置。 + +Diffusers-adapter 并非 native pipeline 的替代实现：它由 model class 选择 SANA-specific hooks。 +特别是 checkpoint 声明的 T2V Diffusers class 在请求 I2V 时必须改装为 Diffusers I2V class，且 +shared `num_frames` 要翻译为 Diffusers 的 `frames`。当前 native 范围只由单 GPU 证明；sequence/ +tensor/CFG parallel、Cache-DiT、TeaCache 和 step execution 未验证，adapter 也不提供 native +continuous batching 或并行能力。 + +## 从输入到输出的主要流程 + +native T2V 解析 prompt、采样参数和 extras，按 profile 的分辨率 bin 处理尺寸，编码文本，初始化 +latents，然后在 checkpoint scheduler 上运行 CFG denoise 并经 VAE 解码视频。真实请求将公共 +image-oriented `num_frames=1` omitted sentinel 解析为 81；profile dummy run 保留一帧。 + +I2V 在同一主干前处理输入 PIL image，按目标尺寸编码它，并把首帧 latent 写入初始 latents； +conditioning mask 令该首帧在 denoise 中固定，同时使用 spatial timestep conditioning。没有 +`multi_modal_data.image`、或传入不支持的 image type/path，必须在 model boundary 明确失败。 + +## 怎样验证功能、精度和性能 + +组件、权重名和 pipeline contract 由 `tests/diffusion/models/sana_video/` 覆盖。SANA I2V 不进入 +通用 tiny alignment harness（该 harness 不提供 image task）；专有 component tests 和 L4 +`tests/e2e/online_serving/test_sana_video_expansion.py` 覆盖真实权重的 native/adapter、T2V/I2V、 +480p/720p matrix。golden 与逐 stage alignment 仅用于调查数值变化，运行前需要其外部 artifact/ +硬件前提，不能把收集或跳过当作完整功能验证。480p T2V 的 frozen golden 只使用 case-specific +0.91 SSIM gate；其余三例保持 0.93。该差异从 native prompt cross-attention 的 vLLM-Omni +Attention 与 reference SDPA 边界开始，不能据此放宽其他 case。^[PR #5508] From bf9697be1bd0e14022a5c800d0459952a6c8af9c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:36:54 +0800 Subject: [PATCH 287/465] knowledge(vllm-omni): sync 53b60bd9 (PR #5670) Upstream-Commit: 53b60bd9bcbbebc49fe75792dfc668580819ffe2 Upstream-PR: #5670 Knowledge-Delta: advance audited pins for the already-distilled content-aware reference-audio cache contract --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- knowledge/repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 624c25bd5..68938d714 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4af0ddc1c2cf007dbbe69d9abab9f415dde805e6 + previous_audited_sha: 50b8a150f9df5ec17ff3a1a99ab113eb57d786d8 audited_ref: main - audited_sha: 50b8a150f9df5ec17ff3a1a99ab113eb57d786d8 + audited_sha: 53b60bd9bcbbebc49fe75792dfc668580819ffe2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0831ec867..3814b03d7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -50b8a150f9df5ec17ff3a1a99ab113eb57d786d8`. The canonical machine baseline +53b60bd9bcbbebc49fe75792dfc668580819ffe2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index afc3c198a..0764b2e8f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 50b8a150` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 53b60bd9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index fd3a59d1f..b21b1c79a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 50b8a150` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 53b60bd9` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 29c55d672..d6c5c72bf 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 50b8a150` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 53b60bd9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 05024dc1e..097870558 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 50b8a150` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 53b60bd9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 01da16cea..92830e8cc 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 50b8a150` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 53b60bd9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 86abd8977..456fe20a9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 50b8a150` 验证存在 +- 源码校验：以上路径均已在 `main @ 53b60bd9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 66993c4d6..75851f7d3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 50b8a150`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 53b60bd9`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From eb47692da2f050f77a249c122b473357b1ebff75 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:42:43 +0800 Subject: [PATCH 288/465] knowledge(vllm-omni): sync 710024fa (PR #6222) Upstream-Commit: 710024fa6fa6ce947045a5d8d1695ecb00d7801d Upstream-PR: #6222 Knowledge-Delta: capture LongCat Image Edit request-geometry precedence and propagation --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../vllm-omni/models/longcat-image/_index.md | 15 ++++++-- .../vllm-omni/models/longcat-image/rules.md | 35 +++++++++++++++++++ 11 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/longcat-image/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 68938d714..e5a675c01 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 50b8a150f9df5ec17ff3a1a99ab113eb57d786d8 + previous_audited_sha: 53b60bd9bcbbebc49fe75792dfc668580819ffe2 audited_ref: main - audited_sha: 53b60bd9bcbbebc49fe75792dfc668580819ffe2 + audited_sha: 710024fa6fa6ce947045a5d8d1695ecb00d7801d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3814b03d7..3c566dd9b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -53b60bd9bcbbebc49fe75792dfc668580819ffe2`. The canonical machine baseline +710024fa6fa6ce947045a5d8d1695ecb00d7801d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0764b2e8f..42434c72b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 53b60bd9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 710024fa` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b21b1c79a..dae95a643 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 53b60bd9` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 710024fa` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d6c5c72bf..3bdf2eb03 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 53b60bd9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 710024fa` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 097870558..9fc824717 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 53b60bd9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 710024fa` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 92830e8cc..dd72c0b3b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 53b60bd9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 710024fa` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 456fe20a9..1bbf45c74 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 53b60bd9` 验证存在 +- 源码校验：以上路径均已在 `main @ 710024fa` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 75851f7d3..6539f8582 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 53b60bd9`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 710024fa`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/longcat-image/_index.md b/knowledge/repos/vllm-omni/models/longcat-image/_index.md index ff32ae077..764742951 100644 --- a/knowledge/repos/vllm-omni/models/longcat-image/_index.md +++ b/knowledge/repos/vllm-omni/models/longcat-image/_index.md @@ -1,7 +1,7 @@ --- title: "LongCat-Image（Qwen2.5-VL 编码 T2I + 编辑）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] sources: [vllm_omni/diffusion/models/longcat_image/, vllm_omni/diffusion/registry.py] @@ -9,7 +9,7 @@ sources: [vllm_omni/diffusion/models/longcat_image/, vllm_omni/diffusion/registr # LongCat-Image -以下事实在 `main @ 5d44868e` 复核（源码派生页,尚无本模型的运行经验沉淀）。 +以下事实在 `main @ 710024fa` 复核。 ## 名称与范围 @@ -33,7 +33,18 @@ sources: [vllm_omni/diffusion/models/longcat_image/, vllm_omni/diffusion/registr 时点名核对这两个 import 常量。 - `FlowMatchEulerDiscreteScheduler` + `AutoencoderKL`;CFG-parallel;hub 子目录预取。 +- Edit 的 resolved request geometry 是一条端到端合同：显式 `height`/`width` + 优先于按 source aspect ratio 计算的默认值，随后按 `vae_scale_factor * 2` 向下对齐 + （最小为一个 alignment）。同一对尺寸同时驱动 conditioning image、half-size prompt + image、VAE/packed latents、position IDs 与 decode，不能保留一条独立的 source-derived + generation geometry；具体审查门禁见本目录的 Edit geometry rule。 ## 什么时候查这里 - 审查 longcat 的双语系统提示、Edit 前处理或编码链改动。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| Edit 的 `height`/`width`、conditioning resize 或 latent geometry 改动 | [Edit geometry rule](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/longcat-image/rules.md b/knowledge/repos/vllm-omni/models/longcat-image/rules.md new file mode 100644 index 000000000..aa71c5588 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/longcat-image/rules.md @@ -0,0 +1,35 @@ +--- +title: "LongCat-Image 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #6222", vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py] +confidence: high +--- + +# LongCat-Image 规则 + +只有 `LONGCAT-数字字母` 是可审计规则 ID。 + +## LONGCAT-1a — Edit 请求几何必须单一解析并端到端传播 + +- 触发：修改 `LongCatImageEditPipeline` 的 sampling `height`/`width`、conditioning + preprocess、latent preparation 或 decode geometry。 +- 强制：先解析一对 request geometry（显式 sampling override 优先，否则按输入图像 + aspect ratio 的默认面积计算），以 `vae_scale_factor * 2` 做 minimum-aligned + normalization，并将该同一对尺寸写回 sampling params、用于 conditioning image 与 + half-size prompt image resize、`check_inputs`、latent/position-ID preparation 和 decode。 +- 禁止：在 override 已解析后继续把 source-derived `calculated_height`/`calculated_width` + 传给 conditioning 或 generation；不要把 LongCat Edit 当作可让 conditioning 与 noise + latents 使用不同空间 geometry 的编辑 pipeline。 +- 验收：以非正方形 source 与显式目标尺寸运行 I2I，断言 aligned resolved H/W 同时到达 + preprocess、condition/noise latent packing 和输出；再覆盖未设 override 的 source-AR + default，以及不会归零的极小或非对齐尺寸。当前 common tiny case 的 source/target 都是 + 512×512，只证明旧的 1MP override 失效会被抓到；它没有覆盖非正方形或非对齐 override。 + TeaCache 与 SP+Cache-DiT 也属于不同 acceleration group，不能据此声称 TeaCache+SP 已验证。 + ^[PR #6222] + +共享 diffusion request/latent 生命周期见 +[Diffusion rules](../../components/diffusion/rules.md)，模型结构入口见 +[LongCat-Image index](_index.md)。 From cc4c7d7d53e0ba0086b892540e4ff2c56ec44560 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:51:17 +0800 Subject: [PATCH 289/465] knowledge(vllm-omni): sync 702a4096 (PR #6021) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../vllm-omni/components/distributed/rules.md | 9 ++++++++- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-bridge-batch.md | 6 ++++++ .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 8 +++++++- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../vllm-omni/models/minicpm-o-4-5/_index.md | 3 ++- .../minicpm-o-4-5/rules-code2wav-batching.md | 19 +++++++++++++++++++ 14 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-code2wav-batching.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e5a675c01..bc4bc0c50 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 53b60bd9bcbbebc49fe75792dfc668580819ffe2 + previous_audited_sha: 710024fa6fa6ce947045a5d8d1695ecb00d7801d audited_ref: main - audited_sha: 710024fa6fa6ce947045a5d8d1695ecb00d7801d + audited_sha: 702a409664992df0e5b2d4e3884a88d526ec678e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3c566dd9b..730cc9c1b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -710024fa6fa6ce947045a5d8d1695ecb00d7801d`. The canonical machine baseline +702a409664992df0e5b2d4e3884a88d526ec678e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 42434c72b..e3464a842 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 710024fa` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 702a4096` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index dae95a643..198c3e174 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 710024fa` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 702a4096` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3bdf2eb03..69e5f37c1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 710024fa` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 702a4096` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 4b9540ce2..d83dcd486 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6033", "PR #6360", "PR #6406"] +sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406"] confidence: high --- @@ -70,6 +70,13 @@ confidence: high - 禁止：把未标记的 generation/diffusion payload 改成替换语义；把 control-only boundary 写成未就绪的空更新；跨 segment merge/replay 旧 terminal audio。 - 验收：覆盖标记/未标记、audio+ref、空 replacement 和重复 terminal；断言新 prompt、finished/ready 状态正确，旧 audio 不会进入下一次 Code2Wav。 ^[PR #6406] +## DIST-1g — 异步 processor 任务必须在入队时冻结边界状态 + +- 触发：`OmniChunkTransferAdapter.save_async` 向 custom next-stage processor 入队，且 producer request 会在 worker 发送前继续被 streaming update 原地修改。 +- 强制：只为 custom processor shallow-copy request，并递归复制其可变容器输入（additional information、prompt/token lists）；tensor leaves 保持共享。每个 queued segment boundary 在入队时移除 sender dedup watermark，使下一个 segment 能从零建立自己的 watermark；后台发送旧 task 不得清除新 segment 的 watermark。 +- 禁止：给无 hook 的直通路径增加 request snapshot/copy；让 queued task 读取后来 mutation 的 token、metadata 或 output history；在后台 terminal cleanup 无条件删除可能已属于下一 segment 的 watermark。 +- 验收：old/new segment 在 background flush 前交错时，新 watermark 保持；连续入队后各 processor 观察各自 admission-time request snapshot；无 custom processor 时 task 仍引用原 request。^[PR #6021] + ## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 - 触发：上游 MoE factory/oracle 或 `init_model_parallel_group` 增加 all-to-all manager 要求。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9fc824717..792477253 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 710024fa` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 702a4096` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md index 34e6b369f..fa5006623 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -160,3 +160,9 @@ Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x - 禁止：让共享 runner 默认改用全量替换；把旧 terminal payload 合并回新 snapshot；只修 scheduled-new 而遗漏 cached admission；把 runner bookkeeping 当作模型 payload。 - 验收：覆盖 new/cached replacement、未标记增量 merge、request mirror 和 sibling request 隔离，并确认 NPU 继承路径使用同一替换实现。 ^[PR #6406] +## EXEC-1q — ragged stateful batch 必须维持输入行和 request-owned cache 的一一对应 + +- 触发：stateful generation/codec consumer 放宽 batch compatibility key，令同一 batch 可含不同当前 chunk 长度或 final flag。 +- 强制：保留 input-row 顺序，按 row 维护下一状态；短行可在一次 masked/padded compute 中执行，超出既有 encoder window 的行必须回退到已有的 safe sliced path。任何缺失 audio/state row 都 fail closed，不能 filter 后压缩结果；平台 fast path 在不支持 ragged mask/valid-length 时必须回退。 +- 禁止：按 batch 位置共享或错配 session cache；让 ragged path 绕过 RelPos/encode slicing；用过滤 `None` 的方式掩盖 hole，或把模型特有的 coalescing wait 写进 generic scheduler。 +- 验收：混合长度、final flag、长行和正常行，逐 request 对照 exact path 的 audio/cache；注入中间缺行应报错而非重排。模型专有的 Code2Wav cache/dtype contract 见 MiniCPM-o 4.5 rules。^[PR #6021] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index dd72c0b3b..7f24205d8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 710024fa` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 702a4096` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 1c951ab5c..760b8c620 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6033", "PR #6360", "PR #6406"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6360", "PR #6406"] --- # Scheduler 规则 @@ -285,3 +285,9 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 禁止：只更新 prompt 而继承旧 cache/watermark；把同一 in-flight frame 在更新和 replacement 路径重复计入 stale；绕过 scheduler admission 直接恢复运行。 - 验收：覆盖 running replacement、ready async replacement 的重复 reset 和正常重新调度；断言 KV/encoder 释放各一次、watermark 清零、stale counter 恰好可排空且首个新 segment frame 不被丢弃。 ^[PR #6406] +## SCHED-6e — 流式 segment 边界必须在 request mutation 前冻结发送 watermark + +- 触发：`OmniARScheduler.update_from_output` 停止 resumable async-chunk request，且同一处理周期可能由 streaming update 替换或重置该 mutable `Request`。 +- 强制：在 `_handle_stopped_request` 及任何会重置 computed-token 状态的 transition 前，从 chunk adapter 取得 confirmed computed-token watermark；将该冻结值显式传给 `save_async`。后续 segment 只能建立自己的 watermark，不能让旧边界任务读取已重置 request。 +- 禁止：在 request replacement 后重新计算旧 segment 的 send watermark；以当前 `num_computed_tokens` 推断已确认旧边界；把模型专用 Code2Wav batching/wait policy 塞进通用 generation scheduler。 +- 验收：模拟 stop handler 将同一 request 重置为新 segment，断言 queued old boundary 使用 transition 前的 confirmed token 数；并验证 generation scheduler 没有新增 MiniCPM/Code2Wav-specific coalescing state。^[PR #6021] diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1bbf45c74..d0bb8fd8d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 710024fa` 验证存在 +- 源码校验：以上路径均已在 `main @ 702a4096` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6539f8582..d97740c55 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 710024fa`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 702a4096`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 5ae8e3204..39b887725 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -49,7 +49,8 @@ Daily-Omni、Seed-TTS simplex/realtime duplex 已进入 accuracy/perf harness； Online serving CI 的 core suite 固定 async chunk，expansion 分开覆盖 sync/async，native duplex fixture 不再覆盖 eager/KV cap 并固定 active-speech turn window；验证证据边界见 MCPMO-5b。 -描述直达源码与模型专有门禁见 [rules](rules.md#direct-代码快速入口)；新模型语义验证见 +描述直达源码与模型专有门禁见 [rules](rules.md#direct-代码快速入口)；并发 duplex 的 ragged +Code2Wav state、batch cap 与 NPU residual limits 见 [Code2Wav 并发批处理规则](rules-code2wav-batching.md)。新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-code2wav-batching.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-code2wav-batching.md new file mode 100644 index 000000000..349af310d --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-code2wav-batching.md @@ -0,0 +1,19 @@ +--- +title: "MiniCPM-o 4.5 Code2Wav 并发批处理规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, model-executor] +sources: ["PR #6021", vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/platforms/npu/test_graph_tools.py] +confidence: high +--- + +# MiniCPM-o 4.5 Code2Wav 并发批处理规则 + +## MCPMO-1j — Code2Wav 并发 duplex batch 必须按 request 保存 ragged state + +- 触发：修改 `MiniCPMO45Code2Wav` bucket/initial-batch partition、`BatchedToken2Wav.decode_ragged_batch`、attention cache dtype 或 NPU Code2Wav patch。 +- 强制：compatibility bucket 只由 prompt/cache history/epoch 决定；当前 token length 与 final flag 不同的短行可共享 masked ragged DiT pass，输出和 Flow/HiFT cache 必须按原 request row 回填。超出 RelPos-safe encode window 的行回退 `decode_batch` 的 sliced path。`code2wav_initial_batch_size` 为非零时必须不小于 min batch，partition 后每个 initial batch 都满足 min；未设置时不改变既有全 bucket 行为。BF16 attention cache 只改变存储表示，compute 保持 FP32。 +- 禁止：用 batch index 复用另一 session 的 state；让 ragged path 跳过 long-input slicing、final-row mask 或 complete-row 检查；把 BF16 cache 当作 compute dtype 变化；用 generic generation scheduler 等待来凑 MiniCPM batch，或把外部 N=8 证据当作默认 two-session deploy/strict realtime claim。 +- 现有边界：NPU patch 尚未在 `decode_ragged_batch` 外围安装与 exact path 相同的 flow execution context；其 BF16 guard 使用 Python `bool`，因此字符串 `"false"`/`"0"`/`"off"` 会被误判为启用；当 initial cap 为 0 且 min batch 大于 1 时 initial-marker bucket 仍可绕过 min check；steady-state batch 没有 execution-layer cap。CUDA-only BF16 opt-in 与 NPU/ragged 配置必须分别验证，不能把这些限制描述成已关闭。 +- 验收：比较 mixed-length/final-flag batch 与逐 request exact decode 的 audio、Flow 和 HiFT state；覆盖 oversized row fallback、缺失中间 row fail-closed、initial min/max partition、BF16 storage/FP32 compute，以及上述 NPU/initial-marker/steady-state residual limits 的回归或显式拒绝。^[PR #6021] From 9d3254206d36c16f49c4e28e6b2be45a5f92534c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 17:56:00 +0800 Subject: [PATCH 290/465] knowledge(vllm-omni): sync f0808c97 (PR #5277) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 10 +++++++++- knowledge/repos/vllm-omni/components/serving/rules.md | 4 ++-- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 21 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bc4bc0c50..7cefd4c22 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 710024fa6fa6ce947045a5d8d1695ecb00d7801d + previous_audited_sha: 702a409664992df0e5b2d4e3884a88d526ec678e audited_ref: main - audited_sha: 702a409664992df0e5b2d4e3884a88d526ec678e + audited_sha: f0808c97939f56d6a8e4fa8b910e2605b0073b49 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 730cc9c1b..a27108f27 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -702a409664992df0e5b2d4e3884a88d526ec678e`. The canonical machine baseline +f0808c97939f56d6a8e4fa8b910e2605b0073b49`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e3464a842..0e95ba696 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 702a4096` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f0808c97` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 198c3e174..e1475134e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 702a4096` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f0808c97` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 69e5f37c1..85aae0809 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 702a4096` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f0808c97` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 792477253..f55872f08 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 702a4096` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f0808c97` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7f24205d8..80a0d4a40 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 702a4096` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f0808c97` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d0bb8fd8d..5cc30e9ab 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 702a4096` 验证存在 +- 源码校验：以上路径均已在 `main @ f0808c97` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index b785a77ee..9a62296e6 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py] confidence: high --- @@ -157,6 +157,14 @@ confidence: high timeout 生效；abort success 后才清理 frontend state，orchestrator error 与 timeout 时 state 保留。 ^[PR #6084] ^[PR #6581] +### SERV-5n — remote replica attach/detach 必须按本次 membership generation 判定 + +- 触发：修改 `MembershipController` 的 register、watcher snapshot、unregister、shutdown 或 remote client factory。 +- 强制：以 `(stage_id, replica_id)` 分别记录 attaching/attached，重复 register 在 attach 完成前后都只保留一个 client。每次 watcher snapshot 单调递增 generation；controller 创建的地址只有在本次 attach 开始后真实观察到 `UP`，才可因后续消失触发 detach；未跟踪的 static client 继续按相邻 snapshot 的 `UP -> absent` 清理。factory 返回后若 shutdown 已开始，或地址校验/`StagePool.add_client` 失败，必须关闭尚未接管的 client；正常 detach 同步清理 membership bookkeeping、pool client、受影响请求和 client 资源。 +- 禁止：让 attach 前的 stale `UP` snapshot 立即删除刚注册 client；并发重复创建同一 replica；shutdown 后继续接受 register 或挂入 late client；把“曾在任意 generation 出现”当作当前 attachment 已确认。 +- 已知边界：`asyncio.to_thread` 中正在执行的 factory 不能随外层 task cancellation 一起停止；若 membership task 被直接取消，thread 后来返回的 client 仍可能失去 cleanup owner。该低严重度泄漏在 PR 中明确延期，现有 shutdown/late-result 回归不能证明 cancellation 路径已关闭。 +- 验收：覆盖 register storm、register 前后 stale/fresh `UP`、attach 中 watcher churn、untracked disappearance、invalid stage/address、`add_client` 异常、shutdown-before/during/after factory、detach 后重新注册和受影响 request error；另以专门用例保留或修复上述 cancellation leak。^[PR #5277] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index cd7b92e9f..8a83c7dc9 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -24,7 +24,7 @@ confidence: high | `chat_template_kwargs`、raw HTTP/SDK `extra_body`、text/audio modalities、choices、空音频 | `chat-multimodal-contract`：`SERV-4c` + 命中模型规则 | upstream `ChatCompletionRequest` → `serving_chat.py::{OmniOpenAIServingChat._preprocess_chat,OmniOpenAIServingChat.chat_completion_full_generator,OmniOpenAIServingChat._create_text_choice,OmniOpenAIServingChat._create_audio_choice}` | | endpoint restriction、route/app-state guard、capability、公开 400 | `endpoint-capability`：`SERV-4c`, `SERV-4d`, `SERV-5d` | endpoint policy → `api_server.py::build_app` assembled app → public handler | | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup acknowledgment | `engine-lifecycle`：`SERV-5a`, `SERV-5b`, `SERV-5m` | `entrypoints/async_omni.py::{AsyncOmni.pause_generation,AsyncOmni.resume_generation,AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni._abort}` → `engine/{stage_pool,orchestrator,async_omni_engine}.py` → AR EngineCore helper / diffusion worker RPC | -| stage/replica death、`EngineDeadError`、`StageUnavailableError`、health/readiness、eviction | `engine-lifecycle`：[SERV-5f](rules-fault-isolation.md#serv-5f-stage-death-按-replica-隔离request-fatal-不等于-process-fatal) | `engine/stage_pool.py` live set → `engine/orchestrator.py` poll/dispatch/cleanup → `entrypoints/{omni_base,async_omni}.py` error transport | +| stage/replica death、remote membership register/watch/detach、`EngineDeadError`、`StageUnavailableError`、health/readiness、eviction | `engine-lifecycle`：[SERV-5f](rules-fault-isolation.md#serv-5f-stage-death-按-replica-隔离request-fatal-不等于-process-fatal)、[SERV-5n](rules-engine-lifecycle.md#serv-5n-remote-replica-attachdetach-必须按本次-membership-generation-判定) | `engine/membership_controller.py` register/watcher generation → `engine/stage_pool.py` live set → `engine/orchestrator.py` poll/dispatch/cleanup → `entrypoints/{omni_base,async_omni}.py` error transport | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | | request-level LoRA、AR-only/multistage、streaming input update、stage cardinality | `request-contract`：`SERV-4l` | `serving_chat.py` LoRA resolve → `AsyncOmni.generate` → `AsyncOmniEngine._build_add_request_message` → stage-0 input processor | @@ -43,7 +43,7 @@ confidence: high | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | -| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、factory 状态矩阵、TTS adapter detection、replica fault isolation | `SERV-5a`–`SERV-5e` 与 `SERV-5m`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | +| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5m`–`SERV-5n`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints | `SERV-4a`–`4h`, `SERV-4l`–`4o`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d97740c55..31f7056e5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 702a4096`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f0808c97`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From da45f8cfb71bc60341538a771060d4d036dc1bf2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:01:44 +0800 Subject: [PATCH 291/465] knowledge(vllm-omni): sync bae58f84 (PR #4092) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 16 +++++++++++++++- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 10 files changed, 25 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7cefd4c22..2b3bd6e08 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 702a409664992df0e5b2d4e3884a88d526ec678e + previous_audited_sha: f0808c97939f56d6a8e4fa8b910e2605b0073b49 audited_ref: main - audited_sha: f0808c97939f56d6a8e4fa8b910e2605b0073b49 + audited_sha: bae58f84b86a8e683f87ffabdce3bd3c80e4e788 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a27108f27..05ca5c3cf 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f0808c97939f56d6a8e4fa8b910e2605b0073b49`. The canonical machine baseline +bae58f84b86a8e683f87ffabdce3bd3c80e4e788`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0e95ba696..92ea404eb 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f0808c97` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bae58f84` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e1475134e..07027300a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f0808c97` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bae58f84` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 85aae0809..da2ccc184 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f0808c97` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bae58f84` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f55872f08..785ac11cc 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f0808c97` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bae58f84` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 80a0d4a40..d29495375 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f0808c97` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bae58f84` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5cc30e9ab..aec506875 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f0808c97` 验证存在 +- 源码校验：以上路径均已在 `main @ bae58f84` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 9a62296e6..a64a23288 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py] confidence: high --- @@ -165,6 +165,20 @@ confidence: high - 已知边界：`asyncio.to_thread` 中正在执行的 factory 不能随外层 task cancellation 一起停止；若 membership task 被直接取消，thread 后来返回的 client 仍可能失去 cleanup owner。该低严重度泄漏在 PR 中明确延期，现有 shutdown/late-result 回归不能证明 cancellation 路径已关闭。 - 验收：覆盖 register storm、register 前后 stale/fresh `UP`、attach 中 watcher churn、untracked disappearance、invalid stage/address、`add_client` 异常、shutdown-before/during/after factory、detach 后重新注册和受影响 request error；另以专门用例保留或修复上述 cancellation leak。^[PR #5277] +### SERV-5o — wake 的设备同步只在 device-owning worker 执行 + +- 触发：修改 `AsyncOmni` 的 wake RPC、worker `handle_wake_task()`，或为 sleep/wake + 增加 platform/CUDA 同步。 +- 强制：`AsyncOmni` 只 dispatch wake RPC、等待 ACK 并更新 frontend 状态；实际 device + 同步留在执行 wake 的 worker 路径。这样 proxy 或 HTTP-server-only 的 `AsyncOmni` 不会因 + wake bookkeeping 初始化 CUDA context。 +- 禁止：在 `AsyncOmni` 的 wake completion 路径重新导入 platform 或调用全局 + `synchronize()`；不能以收到 ACK 代替 worker 已完成其设备同步的合同。 +- 验收：用不拥有 CUDA device 的 proxy/frontend 实例经过 diffusion wake RPC，断言 + frontend 不创建 CUDA context；另断言 worker `handle_wake_task()` 在返回成功 ACK 前仍完成 + 同步。PR #4092 只声称既有测试通过，未提供可复现命令或该 proxy 回归；其要求 inline + guard/comment 的 review thread 在合并时仍未 resolve，故这条验收尚未被该 PR 证明。^[PR #4092] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 31f7056e5..06d2f907e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f0808c97`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bae58f84`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 39d18421617973d050e26cdc6efabf3dd57fb9bb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:13:33 +0800 Subject: [PATCH 292/465] knowledge(vllm-omni): sync b454e349 (PR #6149) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 3 ++- .../configuration/rules-stage-transport.md | 20 +++++++++++++++ .../components/configuration/rules.md | 2 ++ .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 11 +++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 25 ++++++++----------- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 13 files changed, 54 insertions(+), 25 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/configuration/rules-stage-transport.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2b3bd6e08..56663a07e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f0808c97939f56d6a8e4fa8b910e2605b0073b49 + previous_audited_sha: bae58f84b86a8e683f87ffabdce3bd3c80e4e788 audited_ref: main - audited_sha: bae58f84b86a8e683f87ffabdce3bd3c80e4e788 + audited_sha: b454e34966f3d14d1605f29aca19c962d3a16152 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 05ca5c3cf..59d4ccdb5 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bae58f84b86a8e683f87ffabdce3bd3c80e4e788`. The canonical machine baseline +b454e34966f3d14d1605f29aca19c962d3a16152`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 92ea404eb..1d5a831c3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bae58f84` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b454e349` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 @@ -36,6 +36,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] |---|---| | 理解配置从 deploy、CLI、默认 factory 到 structured/legacy config 的稳定边界 | [配置构造架构](architecture.md) | | 根据 PR 描述直达 strict schema、pipeline sampling constraints、deploy/topology、composable strategy 或显存配置的规则组与第一批源码 | [配置开发门禁与代码地图](rules.md) | +| stage full-payload transport capability、topology-owned projection 或 deploy/CLI override rejection | [stage transport capability](rules-stage-transport.md) | | 审计配置来源和多层加工 | [config audit](config-audit-plain-language.md) | | 设计配置归一化与 parity 验证 | [config normalization parity](config-normalization-parity.md) | | 查询 `Omni()` 初始化参数 | [Omni init args](omni-init-args.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-stage-transport.md b/knowledge/repos/vllm-omni/components/configuration/rules-stage-transport.md new file mode 100644 index 000000000..c412697ff --- /dev/null +++ b/knowledge/repos/vllm-omni/components/configuration/rules-stage-transport.md @@ -0,0 +1,20 @@ +--- +title: "Stage transport capability" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, config, scheduler, connectors] +sources: [vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/arg_utils.py, tests/config/test_omni_config.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, "PR #6149"] +--- + +# Stage transport capability + +适用于 pipeline stage 的完整 payload 输入能力、其 config projection，以及 deploy/CLI 对该能力的覆盖。 +调度侧的 admission 和 async sender/receiver 合同见 [Scheduler rules](../scheduler/rules.md#sched-6b-full-payload-admission-是解析后的-downstream-capability)；worker connector 验证见 [Model Executor rules](../model-executor/rules.md#exec-3c-connector-required-stage-必须在-worker-启动前验证-runner-capability)。 + +## VOMNI-CFG-1p — stage transport capability 只能由 pipeline topology 解析 + +- 触发：新增或修改 stage 输入传输、`requires_full_payload_input`、deploy/CLI override 或 engine-args projection。 +- 强制：完整 payload 的消费能力在 `StagePipelineConfig` 中声明，并在 legacy 与 structured 配置构造时无条件覆盖进最终 `OmniStageModelConfig`、`OmniEngineArgs` 与 `OmniModelConfig`；必须保留 `False`，且 deploy 的 flat key、`engine_args`、`engine_extras` 和 CLI 都不能伪造或覆盖该 topology 结论。 +- 禁止：用 architecture/stage 名 allowlist 重新推断能力；让 free-form deploy/CLI 在 producer 校验之后改写 transport；把 async producer 的 `async_chunk` 发送职责误当作下游 full-payload wait capability。 +- 验收：覆盖旧 allowlist 对应的 pipeline declarations 和一个 token-only control；三个 legacy YAML 拼写及 CLI override 均因没有 structured owner 而拒绝；从 pipeline 到最终 model config 断言 true/false 均完整保留。^[PR #6149] diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index a6faf07c0..d5d126c6e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -26,6 +26,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | | multi-stage HF sub-config、stage quantization view、`hf_config_name` | `stage-model-config`：`VOMNI-CFG-1i` | pipeline stage declaration → `OmniModelConfig::{draw_hf_text_config,get_model_arch_config}` → vLLM quantization selection | | pipeline `sampling_constraints`、`StageConfig.to_omegaconf()`、runtime stage config | `stage-config-propagation`：`VOMNI-CFG-1o` | `merge_pipeline_deploy` → `StageConfig.to_omegaconf()` → `engine.stage_configs` | +| stage transport、`requires_full_payload_input`、topology projection 或 override rejection | `stage-transport`：`VOMNI-CFG-1p` | `stage_config.py::{StagePipelineConfig,_build_engine_args}` → `omni_config.py::_build_model_config` → `engine/arg_utils.py::OmniEngineArgs` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -36,6 +37,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a`, `CONF-4c`–`4d`，见 [并行拓扑合同](rules-parallel-topology.md) | | `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | | `stage-config-propagation` | pipeline sampling constraints、StageConfig serialization/runtime config | `VOMNI-CFG-1o` | +| `stage-transport` | full-payload transport capability、topology projection、deploy/CLI override | `VOMNI-CFG-1p`，见 [stage transport capability](rules-stage-transport.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `VOMNI-CFG-0a`, `VOMNI-CFG-0b` | ## 配置归一化与新老路径一致性 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 07027300a..bc89bcb4c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bae58f84` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b454e349` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index da2ccc184..107c36309 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bae58f84` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b454e349` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 785ac11cc..876e17903 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bae58f84` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b454e349` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 1223e6c2b..fa6c9b574 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149"] --- # Model Executor 规则 @@ -17,6 +17,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| | strict stage config、known-fields/projection、service/stage 字段归属、runtime config | `strict-stage-config`：`EXEC-3a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_common_stage_config_kwargs,VllmOmniConfig.from_pipeline_config}` → 真实 startup consumer | +| connector-required stage、worker/runner selection、platform worker capability | `connector-capability`：`EXEC-3c` | `engine/arg_utils.py::OmniEngineArgs.__post_init__` → `worker/omni_connector_validation.py::validate_worker_omni_connector` → selected `worker_cls.model_runner_cls` | | runner `_preprocess`、逐请求 metadata、prefill/decode phase、batch preprocess、MTP | `runner-preprocess`：本页“Runner 到模型的预处理合同” | `vllm_omni/worker/gpu_model_runner.py::{OmniGPUModelRunner._maybe_run_batch_preprocess,_preprocess,_build_model_kwargs_extra,_talker_mtp_forward}` → `vllm_omni/model_executor/models/<命中模型>` consumer | | stage TP/PP/DP、devices、replica、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | @@ -31,6 +32,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru |---|---|---| | `core` | 每次 model-executor 审查 | `EXEC-1a` | | `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | +| `connector-capability` | full-payload/connector stage 的 worker runner validation | `EXEC-3c` | | `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`–`EXEC-1i`，见 [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | | `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`–`EXEC-2c`，见 [loader 合同](rules-loader-contract.md) | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a`, `EXEC-5b` | @@ -63,6 +65,13 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru - 验收：structured 与 legacy stage config 都能读回子目录/状态字段；Audex 等多子目录 checkpoint 做 loader smoke；stateful async-chunk 测试证明容量上限、requeue 和 cleanup。 +### EXEC-3c — connector-required stage 必须在 worker 启动前验证 runner capability + +- 触发：stage 增加完整 payload 输入、downstream processor、explicit connector role，或修改 GPU/NPU/XPU worker/runner 选择。 +- 强制：从 `OmniEngineArgs` 的已解析 `worker_cls`（包括 platform 自动选择）读取 `model_runner_cls`；只要 stage 需要 connector，就要求它是 `OmniConnectorModelRunnerMixin` 的子类，并在初始化前以明确错误拒绝不兼容 worker。所有平台 worker 都应把实际 runner 公开为 `model_runner_cls`，不能仅在 `init_device` 内硬编码构造。 +- 禁止：按 model architecture hard-code connector 初始化 allowlist；等到 request parked 或 connector method 缺失才失败；把无 connector 的普通 async scheduler stage 误判为 connector-required。 +- 验收：覆盖显式兼容/不兼容 worker、platform-resolved worker、无 connector configuration 的 control，以及 GPU、NPU、XPU AR/generation worker 的 runner-class 暴露和 mixin 继承。^[PR #6149] + ### EXEC-4a — full-duplex chunk metadata 必须按 span 隔离且可安全序列化 - 触发：`experimental/fullduplex` 修改 PCM/audio span、force-listen、rollback、 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d29495375..ad19fdc95 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bae58f84` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b454e349` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 760b8c620..6c4cab9aa 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6360", "PR #6406"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6149", "PR #6360", "PR #6406"] --- # Scheduler 规则 @@ -258,25 +258,22 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 ## 相关 -## SCHED-6b — request-end full payload 是显式 admission capability +## SCHED-6b — full-payload admission 是解析后的 downstream capability -- 触发：新增或修改只在 request end 消费完整上游 sequence 的 stage。 -- 强制：coordinator allowlist 与模型 capability 同步；完成前零传输，完成时只 enqueue 一次完整 sequence， - consumer 使用 non-async-chunk topology。IndexTTS 2.5 的精确键是 - `("IndexTTS25S2MelDecoder", "indextts2_5_s2mel_decoder")`。 -- 禁止：按模型名猜测、扩大到整个家族，或逐 token 重复搬运。 -- 验收：allowlisted/non-allowlisted 对照、完成时一次传输、abort 无残留。^[PR #5957] +- 触发：新增或修改只在 request end 消费完整上游 sequence 的 stage，或修改 full-payload waiting coordinator。 +- 强制：以 pipeline 解析的 `requires_full_payload_input` 驱动 stage_id > 0 的 non-async coordinator；async-chunk stage 走 streamed connector，不创建 full-payload wait。完成前零传输，完成时只 enqueue 一次完整 sequence。 +- 禁止：按模型名、architecture/stage key 或旧 allowlist 猜测能力；扩大到整个模型家族；把 stage-0 async sender 因没有 full-payload input 而不初始化 chunk adapter；逐 token 重复搬运完整 payload。 +- 验收：覆盖 stage 0、async-chunk consumer、未声明 consumer 和声明的 sync consumer 四个 gates；另覆盖 stage-0 async sender 与 stage-1 receiver 都保有 adapter、完成时一次传输和 abort 无残留。^[PR #5957] ^[PR #6149] - 机制与边界见 [architecture](architecture.md)；跨 stage 数据面见 [Distributed 组件](../distributed/_index.md)。 -## SCHED-6c — full-payload allowlist 必须精确绑定最终 consumer 与 async wiring +## SCHED-6c — payload transport 的发送与接收能力必须分开验证 -- 触发：新增 async-chunk processor 或把某个 stage 加入 full-payload input allowlist。 -- 强制：allowlist 使用精确的 `(model_arch, stage_key)` consumer 配对；Nemotron 的 `talker -> code2wav` 才能接收完整 code payload，`thinker -> talker` 保持 token-only；sync full-payload builder 与 async-chunk builder 必须分别接入同一真实 code2wav consumer。 -- 禁止：按模型家族或 stage index 猜测 full-payload 能力；把 token-only 的 thinker→talker hop 放进 full-payload allowlist；让非 async deploy 逐 token搬运完整 code stack，或让 async 终端 chunk 缺少 finished/empty-terminal 语义。 -- 验收：覆盖 allowlisted code2wav 与 non-allowlisted talker 对照，分别验证普通 deploy 的一次 request-end 全 payload、streaming deploy 的累计 chunk/终端 chunk、connector requeue 和 abort cleanup。 -^[PR #5842] +- 触发：新增 async-chunk processor、full-payload consumer，或变更 runner connector 初始化。 +- 强制：`requires_full_payload_input` 只声明同步 downstream receive/wait；`async_chunk` 独立保留 producer 与 receiver 的 chunk adapter。connector 需求还可来自 downstream processor hook 或 explicit connector role，且每个选择的 GPU/NPU/XPU worker 的 `model_runner_cls` 必须实现 `OmniConnectorModelRunnerMixin`，在 worker 启动前 fail fast。 +- 禁止：从 consumer capability 倒推出 sender；把 Nemotron token-only thinker→talker 设为 full-payload；让没有 connector mixin 的平台 worker 在运行时才挂起；让非 async deploy 逐 token 搬完整 code stack，或让 async terminal chunk 缺少 finished/empty-terminal 语义。 +- 验收：对一个支持和一个不支持 connector 的显式 worker、以及 platform-resolved worker 断言启动前成功/`ValueError`；验证 GPU、NPU 和 XPU worker 公开正确 runner class，并覆盖 sync request-end payload、streaming 累计/terminal chunk、requeue 和 abort cleanup。^[PR #5842] ^[PR #6149] ## SCHED-6d — 显式 prompt replacement 必须一次性释放旧状态并回到 admission diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index aec506875..d9bb155eb 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bae58f84` 验证存在 +- 源码校验：以上路径均已在 `main @ b454e349` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 06d2f907e..6a408cee5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bae58f84`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b454e349`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From a21c28646a19d53c0b9aa6266cc62d9fd6870a0b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:22:00 +0800 Subject: [PATCH 293/465] knowledge(vllm-omni): sync 51b7565f (PR #6542) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 4 +-- .../components/diffusion/rules-attention.md | 21 +++++++++++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../vllm-omni/models/minimax-h3/_index.md | 6 ++-- .../vllm-omni/models/minimax-h3/rules.md | 35 ++++++++++++------- 12 files changed, 59 insertions(+), 25 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 56663a07e..d0bbcc43a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bae58f84b86a8e683f87ffabdce3bd3c80e4e788 + previous_audited_sha: b454e34966f3d14d1605f29aca19c962d3a16152 audited_ref: main - audited_sha: b454e34966f3d14d1605f29aca19c962d3a16152 + audited_sha: 51b7565f33017d74e2da7045327a65e8f048eaae # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 59d4ccdb5..9bb0d2cda 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b454e34966f3d14d1605f29aca19c962d3a16152`. The canonical machine baseline +51b7565f33017d74e2da7045327a65e8f048eaae`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1d5a831c3..47d18ac67 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b454e349` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 51b7565f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index bc89bcb4c..ef988492d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b454e349` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 51b7565f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -36,7 +36,7 @@ sources: [] | Wan VAE height/width spatial reshard、empty tail、attention extent | [Wan spatial-shard rules](rules-wan-spatial-shard.md) | | multi-DiT、dotted `_dit_modules`、loader-to-offloader handoff 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | -| local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback 与 config propagation | [attention rules](rules-attention.md) | +| local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback、TRTLLM packed-padding 与 config propagation | [attention rules](rules-attention.md) | | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | | video/audio mux、DLO DP wave、result queue、async pump、SHM ownership、async-output wait 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index e1b8ef3f2..cdba44e96 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -56,6 +56,27 @@ confidence: high 没有证明 third-party registry override 或未继承 base class 的 backend 完成了 capability census；这些实现仍须继承合同或显式实现 method。^[PR #5891] ^[PR #5997] +## DIFF-1m — TRTLLM packed-padding 快路径必须隔离 producer contract、ragged documents 与 SAGE gate + +- 触发：修改 `PackedPaddingMetadata`、TRTLLM packed `cu_seqlens`、`supports_packed_mask_free`、 + padded-output restore、SAGE quantization gate 或 multi-document varlen capability。 +- 强制：TRTLLM 只把 `PackedPaddingMetadata` 当作 single physical batch 的 producer-owned + `[real,pad]` shortcut：Q/KV host lengths 必须是 Python int，在 flattened tensors bounds 内并与 + `max_seqlen_q/k` 及可选 `valid_kv_length` 相等；canonical two-element `int32` cu_seqlens 必须和 + Q/K device 一致。该 path 裁 Q/K/V 后把 output 补零回物理 shape，且从不读取或 materialize + `attn_mask`。generic complete cu_seqlens 仍支持 genuine multi-document ragged varlen，并删除 + paired trailing empty document；两条合同不能互相降级或混用。 +- 禁止：接受任何 nonempty TRTLLM `attn_mask`、用 producer shortcut 表示 arbitrary mask 或 + multi-request block-diagonal layout，或让未验证的 CUDA scalar read 回到 single-request fast path。 + 不得把 tail document 短于 `k_block_size` 造成的 dense fallback 描述为 SAGE 的真实-document + 检查已经完成。 +- 验收：backend tests 分别覆盖 malformed host/device/dtype/shape/max-length metadata、trim/zero + restore、generic unequal Q/KV ragged batch、trailing empty pair、complete packed metadata 加 + nonempty mask 的 rejection，以及 real multi-document output 对逐 document reference。SAGE 还要覆盖 + alignment tail 不参与 real-document minimum；该 fix 在 PR #6542 尚未提交，现存行为可整 batch + fallback dense。性能只可引用该 PR 的 exact 4×B300 Nsight all-to-all→FMHA interval，不能以 CPU mock、 + kernel gap 或 frame hash 声称 E2E latency/quality。^[PR #6542] + ## DIFF-1h — RainFusion irregular tail 由 MindIE-SD 保护，不能 padding 伪对齐 - 触发：修改 `RAINFUSION_ATTN`、`rf_v2`、`VideoTokenLayout`、block size、MindIE-SD diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 107c36309..d7001c3eb 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b454e349` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 51b7565f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 876e17903..fd2549004 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b454e349` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 51b7565f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ad19fdc95..6a1db41f9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b454e349` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 51b7565f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d9bb155eb..09412384e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b454e349` 验证存在 +- 源码校验：以上路径均已在 `main @ 51b7565f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6a408cee5..2ba8d0330 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b454e349`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 51b7565f`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index afa91dff6..473335387 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -76,8 +76,10 @@ confidence: high - LightX2V Turbo v1.0 仅支持 legacy dynamic LoRA；精确 artifact mapping、packed QKV/FC1 binding、active-request sampling 合同与 offload 排除项见 MMH3-2j；转换后的 mixed-rank native PEFT checkpoint 不在该支持声明内。 -- datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；packed suffix trim、SAGE 短序列 role、 - mask/metadata fail-closed、AllGather-KV 禁用及性能证据边界见 MMH3-1e。 +- datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；single-request producer-owned + `PackedPaddingMetadata` 可替代 structural suffix mask，generic ragged/continuous batches仍按 + cu_seqlens 隔离 documents。SAGE short-role 与 multi-request tail gate、mask/metadata fail-closed、 + AllGather-KV 禁用及有界 performance evidence 见 MMH3-1e。^[PR #6542] - 2×consumer-GPU profile 使用 TP-local no-AllGather DLO、VAE patch parallel、cuDNN attention 和 eager execution；resident layers 只改变 HBM/transfer，不减少 pinned host master。 实现合同与 standalone-audio staging 缺口见 MMH3-3a，5090/4090 证据边界见 MMH3-3b。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 98c52e887..edb49f908 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -127,26 +127,37 @@ confidence: high - 强制：H3 只在支持的 datacenter Blackwell SM100/103、head_dim=128、FlashInfer kernel 可用且 声明 compatible packed/mask-free path 时默认 dense BF16 TRTLLM；SM120/121、缺依赖、错误 head dim 或需要任意 mask 的路径保留平台 fallback，默认不自动开启 SAGE/Skip-Softmax。 -- 强制：四个 packed metadata key 必须完整、Q/K batch/terminal 覆盖一致；只接受 prefix-valid - structural padding mask，按 boundary 裁掉无效 suffix 后再 quantize/attention，并把输出补零到 - Ulysses 物理 shape。`[0,N,N]` 必须折叠尾部空 sequence；任意 mask、缺失或矛盾 metadata - fail closed。`attention_mask_free=True` 是 compatible packed-path capability，不表示物理 mask - 永远为空。AllGather-KV 的 local-Q/global-KV metadata 不对称合同在初始化时明确拒绝，真正 - 支持按 review 明确延期。 +- 强制：四个 packed metadata key 必须完整、Q/K batch/terminal 覆盖一致。单 request 的 + producer-owned `[real,pad]` path 仅在 non-Ring backend 声明 `supports_packed_mask_free()` 时可省略 + `attn_mask`，并同时发布 Python `q_length`/`kv_length` 与同 device、`int32`、canonical + `[0,length]` 的 `PackedPaddingMetadata` views；TRTLLM 必须核对长度、bounds、max-length 与 + `valid_kv_length` 后裁掉 suffix、再 quantize/attention，并把输出补零到 Ulysses 物理 shape。 + TRTLLM 对任何 nonempty `attn_mask` 一律 fail closed；generic ragged cu_seqlens 仍是多 document + block-diagonal path，`[0,N,N]` 尾部空 sequence 必须折叠。AllGather-KV 的 local-Q/global-KV + metadata 不对称合同在初始化时明确拒绝，真正支持按 review 明确延期。 - 强制：main DiT 与 `minimax_h3.token_refiner` 使用独立 role；任一 KV sequence 短于 SAGE `k_block_size` 时该 input 告警并走 dense TRTLLM，不能让 14-token refiner 产生 non-finite， 也不能因此关闭 main DiT 的 SAGE。H3 将 scheduler 的降序 sigma（1→0）发布为 `normalized_timestep`；Skip-Softmax 在 sigma 大于 `disabled_until_timestep` 时保持 dense， 到达或低于阈值才启用。 -- 禁止：把 structural prefix mask 泛化为任意 mask；让 padding 进入 SAGE block quantization； - 把 combined packed-H3 优化视频归因于本 PR 单一改动。请求级 `set_forward_context` wrapper - 会在 `finally` 恢复 prior context，因此 loop 内的 step mutation 不会泄漏到后续请求。 -- 验收：测试覆盖 ragged batch、suffix trim/zero restore、empty terminal、invalid mask/metadata、 - short-role dense fallback、AllGather rejection 与 non-finite regression。PR 的 4×B300 SM103、 +- 禁止：把 producer-owned packed-padding shortcut 泛化为任意 mask、multi-request batch 或第三方 + metadata；让 padding 进入 SAGE block quantization；把 combined packed-H3 优化视频归因于本 PR + 单一改动。请求级 `set_forward_context` wrapper 会在 `finally` 恢复 prior context，因此 loop 内的 + step mutation 不会泄漏到后续请求。 +- 验收：测试覆盖 generic ragged batch、single-request packed-padding suffix trim/zero restore、empty + terminal、complete packed metadata 加 nonempty mask 的 unconditional rejection、invalid metadata、 + short-role dense fallback、AllGather rejection 与 non-finite regression。Continuous batching 必须 + 覆盖 aligned non-final request 的严格递增 `cu_seqlens`；SAGE gate 应只依据 real documents，当前 + multi-request alignment-tail case 仍可能使整 batch dense，是 PR #6542 留下的 unresolved P3，不能 + 声称 SAGE 保持启用。PR 报告的 4×B300 SXM6、TP1/Ulysses4/Ring1、1344×768、243 frames、49 + updates、dense BF16 TRTLLM、单 warmup 后 Nsight interval 从 final Ulysses all-to-all 到 main FMHA： + 9,797/9,680 paired samples的 gap p50 499.683→0.352 us、p95 698.308→0.384 us，且中间 GPU + activity/D2H copy 为 244,925/97,970→0/0；这只证明该 exact profiling interval，不是 E2E + latency、quality 或其他 workload/backend/topology 的保证。此前 PR 的 4×B300 SM103、 1248×768、209 frames、50 steps FA4/TRTLLM 对比仅绑定该 prompt/seed/topology：PSNR 27.10、 SSIM 0.8880 不是质量 gate；83.854→71.990 s diffusion、88.558→76.176 s wall 的 warmed A/B 来自 review follow-up 外部分支 commit `20cc23ae`，其 artifacts 明确是 combined packed-H3 - optimization evidence，并非目标 commit 的隔离实验，不能泛化为稳定 14% 保证。^[PR #5779] + optimization evidence，并非目标 commit 的隔离实验，不能泛化为稳定 14% 保证。^[PR #5779] ^[PR #6542] ## MMH3-1g — H3 NPU mask-free opt-in 必须保留 Ring 与 malformed-metadata fallback From 2719845603577b1d7b9748ecc97f080bdfac7c87 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:27:37 +0800 Subject: [PATCH 294/465] knowledge(vllm-omni): sync 29e3165b (PR #6343) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- knowledge/repos/vllm-omni/models/wan2-2/architecture.md | 8 +++++++- 11 files changed, 18 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d0bbcc43a..720971d4b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b454e34966f3d14d1605f29aca19c962d3a16152 + previous_audited_sha: 51b7565f33017d74e2da7045327a65e8f048eaae audited_ref: main - audited_sha: 51b7565f33017d74e2da7045327a65e8f048eaae + audited_sha: 29e3165bb938088d007cad9094cf66154c8d88c1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9bb0d2cda..37c70503b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -51b7565f33017d74e2da7045327a65e8f048eaae`. The canonical machine baseline +29e3165bb938088d007cad9094cf66154c8d88c1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 852bb5c1f..45acd6f64 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 47d18ac67..d2f108657 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 51b7565f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 29e3165b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ef988492d..d9299102c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 51b7565f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 29e3165b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d7001c3eb..3b4fd4654 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 51b7565f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 29e3165b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index fd2549004..6c1c4f88d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 51b7565f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 29e3165b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6a1db41f9..1490a9743 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 51b7565f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 29e3165b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 09412384e..a12245928 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 51b7565f` 验证存在 +- 源码校验：以上路径均已在 `main @ 29e3165b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2ba8d0330..848e46904 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 51b7565f`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 29e3165b`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/wan2-2/architecture.md b/knowledge/repos/vllm-omni/models/wan2-2/architecture.md index b62f5f8e8..6b204a679 100644 --- a/knowledge/repos/vllm-omni/models/wan2-2/architecture.md +++ b/knowledge/repos/vllm-omni/models/wan2-2/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-21 updated: 2026-09-02 type: architecture tags: [vllm-omni, models, diffusion] -sources: ["PR #2783", "PR #5969", "PR #4820", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py, vllm_omni/diffusion/models/dmd2/mixin.py] +sources: ["PR #2783", "PR #5969", "PR #4820", "PR #6343", vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py, vllm_omni/diffusion/models/dmd2/mixin.py, tests/dfx/stability/tests/test_wan22.json] --- # Wan 2.2 架构 @@ -82,6 +82,12 @@ e2e（t2v 离线/在线、W4A16）、**accuracy** （perf/reliability/stability）。上列家族测试中未见 DMD2 专属测试（未做穷举 核对）。 +- stability JSON 将 A14B I2V 的两卡 CUDA case 标记为 `H800`，并单列两卡 NPU + `A3` case（`usp=2`、VAE patch parallel=2、HSDP、VAE slicing）；nightly 必须以 + 对应硬件 pytest marker 选择，不能从同一文件的模型或 `diffusion` family marker 推断 + 可在另一后端运行。这些配置只声明收集和启动参数，不构成任何 24-hour 稳定性运行已通过的 + 证据。^[PR #6343] + - 树内已无原 Wan2.2 ModelOpt FP8 exporter，也没有该模型专用的 quantization export example； 审查与验证不得继续引用已删除的 example 路径。副本数据并行示例仍为 `examples/online_serving/replica_data_parallel/wan2_2_ti2v_dp.yaml`。^[PR #5969] From 57b605f342c583f631b4a4fe754d1a8354677880 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:33:11 +0800 Subject: [PATCH 295/465] knowledge(vllm-omni): sync bc8ea489 (PR #6525) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/configuration/deploy-yaml.md | 4 ++++ .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 10 +++++----- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 19 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 720971d4b..f7d3cca93 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 51b7565f33017d74e2da7045327a65e8f048eaae + previous_audited_sha: 29e3165bb938088d007cad9094cf66154c8d88c1 audited_ref: main - audited_sha: 29e3165bb938088d007cad9094cf66154c8d88c1 + audited_sha: bc8ea489effb5fc7aa3934ceb9f915a27765a40d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 37c70503b..cd55d3b68 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -29e3165bb938088d007cad9094cf66154c8d88c1`. The canonical machine baseline +bc8ea489effb5fc7aa3934ceb9f915a27765a40d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d2f108657..1c435333f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 29e3165b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bc8ea489` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md b/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md index aeddcd900..23d15db7e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md +++ b/knowledge/repos/vllm-omni/components/configuration/deploy-yaml.md @@ -34,6 +34,10 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #5647", vllm_omni/deplo （[ci-gotchas](../../ci/guides/ci-gotchas.md) 第 2 条）。 - KV 记账外分配的模型考虑 `kv_cache_memory_bytes` pin（[CONF-2a](rules.md)）。 - 争议以展开后最终配置为准（[CONF-3a](rules.md)）。 +- `diffusion_batch_size` 是客户端请求 batch width，不能替代 scheduler 的 + `max_num_seqs`。需要 Wan2.2 request-level batching 时，在 CLI 显式传 + `--max-num-seqs N` 或在 stage YAML 设置该字段；`wan2_2_ti2v.yaml` 的默认宽度为 4。 + 不要依赖 stage init 把前者写进后者。^[PR #6525] ## 代表样例（79 份 YAML 中的三类拓扑） diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d9299102c..1ef568c99 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 29e3165b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ bc8ea489` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3b4fd4654..389e1b384 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 29e3165b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bc8ea489` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6c1c4f88d..31380cd92 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 29e3165b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bc8ea489` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1490a9743..d8eff6840 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 29e3165b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bc8ea489` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a12245928..a684d9aff 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 29e3165b` 验证存在 +- 源码校验：以上路径均已在 `main @ bc8ea489` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index a64a23288..0e9150c8b 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py] confidence: high --- @@ -248,9 +248,9 @@ confidence: high - 禁止：只解析或记录 `runtime.env` 却不应用到本地 launch；把 stage 环境永久污染到父进程；用 runtime env scope 替代 diffusion device scope；将该本地 scope 推广到 remote replica 或 headless worker launcher。 - 验收：CPU/mock 回归分别覆盖 LLM 和 diffusion launch 看到 stage 值、父环境已有值恢复、原先未设置的 key 在异常后被清理，以及 diffusion launch 同时看到 stage device 值；确认 LLM launch lock 仍覆盖进程创建。^[PR #6214] -### SERV-9a — diffusion stage batch size 必须落到 engine admission capacity +### SERV-9a — client diffusion batch width 不得覆盖 scheduler admission capacity - 触发：diffusion stage 初始化或 replica launch 接收客户端 `batch_size`，或修改其 engine config、scheduler capacity 与 stage startup 路径。 -- 强制：`initialize_diffusion_stage` 和 `launch_diffusion_stage_replica` 在 `build_diffusion_config` 之后、创建 diffusion client/proc manager 之前，都必须将 `od_config.max_num_seqs` 设置为请求的 `batch_size`；direct、parallel 和 single-stage 路径必须使用同一 capacity 语义。 -- 禁止：只把 `batch_size` 传给 client 而保留 config 默认值；只修一个 startup 路径；从 `max_num_seqs > 1` 推断具体 pipeline 已支持 request batching，或绕过 `DIFF-6a` 的条件兼容 admission。 -- 验收：分别 mock `initialize_diffusion_stage` 与 `launch_diffusion_stage_replica`，断言构造出的 `od_config.max_num_seqs` 等于客户端 batch size，并回归 single-stage 初始化；随后以 scheduler 和 Wan2.2 pipeline 测试证明该 capacity 只开放物理槽位，兼容 key、请求隔离和输出拆分仍有效。^[PR #5676] +- 强制：`initialize_diffusion_stage` 和 `launch_diffusion_stage_replica` 只将 `batch_size` 转发给 `StageDiffusionClient`；`build_diffusion_config()` 解析出的 `od_config.max_num_seqs` 必须保持 CLI `--max-num-seqs` 或 stage YAML 的值。两条启动路径和 single-stage 路径都不得把 client `diffusion_batch_size` 映射为 scheduler capacity。 +- 禁止：在 shared init path 写入 `od_config.max_num_seqs = batch_size`，或用 `max(CLI, batch_size)` 合并两个独立旋钮；也不得从 `max_num_seqs > 1` 推断具体 pipeline 自动支持 request batching，兼容 admission 仍受 `DIFF-6a` 约束。 +- 验收：分别 mock `initialize_diffusion_stage` 和 `launch_diffusion_stage_replica`，断言 config 的 `max_num_seqs` 保留 CLI/YAML 的 `1`、`8` 或 `None`，而 client 仍收到 `batch_size`；回归 single-stage 初始化。Wan request-level batching 必须显式设置 `--max-num-seqs N` 或 YAML；Qwen-Image step-execution 的并发 scheduler capacity 不得被默认 `diffusion_batch_size=1` 降为 1。^[PR #5676] ^[PR #6525] ^[Issue #6435] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 848e46904..55d9a36fc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 29e3165b`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bc8ea489`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From f75319a0fb36148b6f61790e37daeee2f803c56d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:36:57 +0800 Subject: [PATCH 296/465] knowledge(vllm-omni): sync d6b66fab (PR #6290) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 10 +++++++++- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 10 files changed, 19 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f7d3cca93..22086258f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 29e3165bb938088d007cad9094cf66154c8d88c1 + previous_audited_sha: bc8ea489effb5fc7aa3934ceb9f915a27765a40d audited_ref: main - audited_sha: bc8ea489effb5fc7aa3934ceb9f915a27765a40d + audited_sha: d6b66faba7c22d1fc99c67d87dcb68ec6e714cb8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index cd55d3b68..725d25e93 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bc8ea489effb5fc7aa3934ceb9f915a27765a40d`. The canonical machine baseline +d6b66faba7c22d1fc99c67d87dcb68ec6e714cb8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index dfde01cde..124d80c0b 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni CI" created: 2026-07-10 -updated: 2026-08-23 +updated: 2026-09-04 type: index tags: [vllm-omni, ci] sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md] @@ -17,6 +17,14 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie - 跨仓库通用的测试方法。 +## Invalid-request error expectation + +- 对 Qwen3-Omni 的 incomplete `response_format={"type": "json_schema"}`，DFX + invalid-request test 应断言 400 公共 API error envelope 中的 + `BadRequestError`、`param="response_format"` 和 `json_schema` 缺失提示；不要把 + vLLM 内部 validator 的 Pydantic `value_error` token 当成该接口合同。该断言曾因 + 上游错误映射改为 `BadRequestError` 而导致 weekly CI 失败。^[PR #6290 / issue #6248] + ## 目录内容 | 遇到什么 | 查看哪里 | diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1c435333f..f2225ff89 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bc8ea489` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d6b66fab` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1ef568c99..62993f3eb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bc8ea489` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d6b66fab` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 389e1b384..540c8f285 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bc8ea489` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d6b66fab` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 31380cd92..7699b58a7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bc8ea489` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d6b66fab` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d8eff6840..c69ab4e38 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bc8ea489` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d6b66fab` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a684d9aff..b1dfee33d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bc8ea489` 验证存在 +- 源码校验：以上路径均已在 `main @ d6b66fab` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 55d9a36fc..788e421c2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bc8ea489`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d6b66fab`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From a7933d2fb6a3dce543428b600128b296ae96743b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:41:12 +0800 Subject: [PATCH 297/465] knowledge(vllm-omni): sync 72beeaee (PR #4807) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../repos/vllm-omni/models/stable-audio/_index.md | 14 ++++++++++++-- 10 files changed, 22 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 22086258f..bc2659670 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bc8ea489effb5fc7aa3934ceb9f915a27765a40d + previous_audited_sha: d6b66faba7c22d1fc99c67d87dcb68ec6e714cb8 audited_ref: main - audited_sha: d6b66faba7c22d1fc99c67d87dcb68ec6e714cb8 + audited_sha: 72beeaee869f19bede9f10077414e1d950845599 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 725d25e93..7fe7be048 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d6b66faba7c22d1fc99c67d87dcb68ec6e714cb8`. The canonical machine baseline +72beeaee869f19bede9f10077414e1d950845599`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f2225ff89..b4b185240 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d6b66fab` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 72beeaee` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 62993f3eb..69cd7166d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d6b66fab` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 72beeaee` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 540c8f285..78fbed7a7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d6b66fab` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 72beeaee` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7699b58a7..17be98a9b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d6b66fab` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 72beeaee` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c69ab4e38..075fa0e43 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d6b66fab` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 72beeaee` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b1dfee33d..0edcc224b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d6b66fab` 验证存在 +- 源码校验：以上路径均已在 `main @ 72beeaee` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 788e421c2..57e7600eb 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d6b66fab`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 72beeaee`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/stable-audio/_index.md b/knowledge/repos/vllm-omni/models/stable-audio/_index.md index 7d23bda53..90dc49146 100644 --- a/knowledge/repos/vllm-omni/models/stable-audio/_index.md +++ b/knowledge/repos/vllm-omni/models/stable-audio/_index.md @@ -1,10 +1,10 @@ --- title: "Stable Audio Open（T2A,cosine DPM-solver）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-08-25 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/stable_audio/, vllm_omni/diffusion/registry.py] +sources: [vllm_omni/diffusion/models/stable_audio/, vllm_omni/diffusion/registry.py, recipes/StabilityAI/Stable-Audio-Open.md, tests/e2e/online_serving/test_stable_audio_online_expansion.py] --- # Stable Audio Open @@ -35,3 +35,13 @@ sources: [vllm_omni/diffusion/models/stable_audio/, vllm_omni/diffusion/registry ## 什么时候查这里 - 审查 stable_audio 的 scheduler wrapper、时长条件或 Oobleck VAE 改动。 + +## 在线发现与覆盖 + +- `main @ 72beeaee` 已删除模型专属的 + `examples/online_serving/stable_audio/` 和对应 User Guide 示例页；不要从这些旧路径寻找 + 可运行示例。当前可发现的使用入口是上游 `recipes/StabilityAI/Stable-Audio-Open.md` 与 + `docs/serving/audio_generate_api.md`；模型目录的 registry 导航仍从 [模型 catalog](../catalog.md) 进入。 +- 在线 E2E 位于 `tests/e2e/online_serving/test_stable_audio_online_expansion.py`：对 gated + `stabilityai/stable-audio-open-1.0` 使用 2 秒、4 steps 的请求，并断言 + `/v1/audio/generate` 返回成功 HTTP 200。它是接口可达 smoke，不是音频字节/WAV 完整性验证。^[PR #4807] From bb6c07763920d6d77b817263c2a7def56b4c489b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:47:01 +0800 Subject: [PATCH 298/465] knowledge(vllm-omni): sync 683f3c92 (PR #6570) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 11 +++++++++++ .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 10 files changed, 21 insertions(+), 10 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bc2659670..c0e2b9bc3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d6b66faba7c22d1fc99c67d87dcb68ec6e714cb8 + previous_audited_sha: 72beeaee869f19bede9f10077414e1d950845599 audited_ref: main - audited_sha: 72beeaee869f19bede9f10077414e1d950845599 + audited_sha: 683f3c92a4750101d132dc61fe4b959fbdb11704 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7fe7be048..815fb6ab4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -72beeaee869f19bede9f10077414e1d950845599`. The canonical machine baseline +683f3c92a4750101d132dc61fe4b959fbdb11704`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 45acd6f64..9cb7d22ea 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -206,6 +206,17 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 workload，而不是声称 perf 是 functional superset。^[PR #6079] ^[PR #5402] ^[PR #5845] +- Qwen3-Omni Async Chunk 的一个 perf JSON 可以承载多个 top-level case；`mark` 是按 case + 附着的，CI 即使使用 `--test-config-file` 也必须用硬件和 scheduling marker 过滤。CUDA nightly + 选择 `H100 and full_model and not slow`，weekly 选择 `H100 and slow`；A3/NPU 保持独立 + `npu` 选择。不要把没有 A3 实测值的 H100 baseline 留在 A3-only case：该 case 在重新测量前 + 只能 collect/run，不能伪装成 A3 阈值。减少 workload 后同样必须移除失效 baseline，待重新测量。 + 清理单例 E2E 前要按输入/输出合同检查 suite-level owner：本轮 default text+audio perf 与 + streaming audio-only 由 multi-replica suite 保留，Async Chunk text-only 仍保留；不支持的 + chat output `audio.format`（如 `aac`）则迁至 weekly invalid-parameter HTTP case，断言 400 和 + supported-format 提示。纯文本五并发的 reduced-token 配置独立覆盖 batching，不能因删除相邻 + audio-only case 而丢失。^[PR #6570] + ## OMNI-CI-3b — patched upstream benchmark 必须保持参数与 output subtype 兼容 - 触发：upstream `benchmarks/serve.py` 增参，或 Omni 聚合字段只存在于扩展 output subtype。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b4b185240..46ef57632 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 72beeaee` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 683f3c92` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 69cd7166d..790817beb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 72beeaee` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 683f3c92` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 78fbed7a7..c3b00757e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 72beeaee` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 683f3c92` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 17be98a9b..0d563c92c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 72beeaee` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 683f3c92` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 075fa0e43..ba6a7d440 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 72beeaee` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 683f3c92` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0edcc224b..67c435b6e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 72beeaee` 验证存在 +- 源码校验：以上路径均已在 `main @ 683f3c92` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 57e7600eb..6024c83e0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 72beeaee`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 683f3c92`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 7d5147a4074651046c1393694cc601486fff4da6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:51:10 +0800 Subject: [PATCH 299/465] knowledge(vllm-omni): sync 0d5d1115 (PR #5544) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 10 +++++++++- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-checkpoint-loading.md | 13 ++++++++++++- .../vllm-omni/components/distributed/_index.md | 4 ++-- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 4 ++-- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 33 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c0e2b9bc3..48e5817bc 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 72beeaee869f19bede9f10077414e1d950845599 + previous_audited_sha: 683f3c92a4750101d132dc61fe4b959fbdb11704 audited_ref: main - audited_sha: 683f3c92a4750101d132dc61fe4b959fbdb11704 + audited_sha: 0d5d1115d6f0b8be86b7b9f003705b6698540887 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 815fb6ab4..9c320e256 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -683f3c92a4750101d132dc61fe4b959fbdb11704`. The canonical machine baseline +0d5d1115d6f0b8be86b7b9f003705b6698540887`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 124d80c0b..a9449d3c3 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544"] --- # vLLM-Omni CI @@ -25,6 +25,14 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie vLLM 内部 validator 的 Pydantic `value_error` token 当成该接口合同。该断言曾因 上游错误映射改为 `BadRequestError` 而导致 weekly CI 失败。^[PR #6290 / issue #6248] +## AutoRound MXFP4 coverage boundary + +- Wan AutoRound MXFP4 has a CPU-marked configuration/layer-mapping test and a full-model + diffusion E2E test gated to XPU B60. The latter uses an environment-overridable model ID, + runs a minimal 256×256 five-frame, one-step request, and checks response success, frame + shape, and nonzero variance. It is an offline inference smoke, not online-serving, FLUX, + CUDA, or general backend coverage. ^[PR #5544] + ## 目录内容 | 遇到什么 | 查看哪里 | diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 46ef57632..d03a11edc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 683f3c92` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0d5d1115` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 790817beb..766b549dc 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 683f3c92` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 0d5d1115` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index fc69573de..4385969f6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6234", "PR #6279", "PR #6445", "PR #6486", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, "PR #5531", "PR #4061"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6234", "PR #6279", "PR #6445", "PR #6486", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #4061"] confidence: high --- @@ -50,6 +50,12 @@ confidence: high - 强制：`ComponentQuantizationConfig.resolve()` 只对 `get_quant_method` 收到的 runtime prefix 做 longest-prefix match，未命中才落到 default；它不会自动感知 checkpoint `WeightsMapper`。 因此 mapper、pipeline 与 layer 必须显式维持同一 namespace，并测试重叠 prefix 的最长匹配。 +- 强制：离线 AutoRound checkpoint 若 `config.json` 以 `data_type="mx_fp"` 声明 MXFP， + `resolve_quant_config_from_disk()` 必须用 checkpoint 的 `bits`（或现有 config 的 + `weight_bits` fallback）重建量化 config，使 4-bit 与 8-bit 路径都由真实位宽选择；不能把 + 这种 metadata 当作只有 serialized flag 才会触发的 online/default config。Wan runtime + prefix 的 `block_name_to_quantize=["blocks"]` 必须只命中 `blocks.*` 的 linear，并保留 + `condition_embedder.*` 等范围外层未量化。 - 禁止：半途裁掉 owner prefix；resolve component 后又要求 ignored layer 带 owner；用一个 组件配置隐式覆盖其他组件；因为同属一个模型就量化全部 linear。 - 验收：至少覆盖“只量化一个组件、其他组件保持 BF16”的真实构造与加载，逐层断言 @@ -58,6 +64,11 @@ confidence: high [MiniMax H3 规则](../../models/minimax-h3/rules.md)；新增 quantization×offload 组合需给兼容矩阵， 未验证 quantizer fail-closed，并核对 dtype/shape/stride。 ^[PR #5136] ^[PR #5737] ^[PR #6279] + 对 AutoRound MXFP4，构造 Wan config 后用 runtime `WanTransformer3DModel` 配置量化层，断言 + packed mapping identity、`blocks.*` 的 MXFP4 method dispatch 与范围外 non-quantized control； + 再以 XPU B60 的 Wan2.1 离线 T2V smoke 验证成功响应、5×256×256 帧形状和非零帧方差。 + 这只证明 PR #5544 的离线 Wan2.1/XPU case；PR body 提到的 FLUX.1-dev 示例及任何 online/ + serving/backend 兼容性均未由合入 diff 的测试覆盖，不得扩展为支持声明。^[PR #5544] ## DIFF-2d — Artifact identity 只信任已验证的不可变来源 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c3b00757e..2d6367e09 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -1,7 +1,7 @@ --- title: "Distributed（跨 stage 通信与数据搬运）" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-04 type: index tags: [vllm-omni, components, distributed] sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coordinator/, docs/design/feature/disaggregated_inference.md] @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 683f3c92` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0d5d1115` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0d563c92c..8694a1a29 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 683f3c92` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0d5d1115` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ba6a7d440..8e73c570c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -1,7 +1,7 @@ --- title: "Scheduler（AR/生成请求调度）" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-04 type: index tags: [vllm-omni, components, scheduler] sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, vllm_omni/core/prefix_cache.py, docs/design/module/ar_runtime.md, docs/design/module/archive/ar_module.md] @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 683f3c92` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0d5d1115` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 67c435b6e..24017ed67 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 683f3c92` 验证存在 +- 源码校验：以上路径均已在 `main @ 0d5d1115` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6024c83e0..86a73ed00 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 683f3c92`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0d5d1115`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 97648292f1e755717ed050d57b6081a32bb86c90 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 18:55:55 +0800 Subject: [PATCH 300/465] knowledge(vllm-omni): sync dd527b42 (PR #6613) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 18 +++++++++++++++++- knowledge/repos/vllm-omni/ci/rules.md | 10 +++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 36 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 48e5817bc..ec74b1174 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 683f3c92a4750101d132dc61fe4b959fbdb11704 + previous_audited_sha: 0d5d1115d6f0b8be86b7b9f003705b6698540887 audited_ref: main - audited_sha: 0d5d1115d6f0b8be86b7b9f003705b6698540887 + audited_sha: dd527b42456ad29d5df4fdbfa56040b3c196a75f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9c320e256..4db550280 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0d5d1115d6f0b8be86b7b9f003705b6698540887`. The canonical machine baseline +dd527b42456ad29d5df4fdbfa56040b3c196a75f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index a9449d3c3..4b0016dba 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6613", tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json] --- # vLLM-Omni CI @@ -33,6 +33,22 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie shape, and nonzero variance. It is an offline inference smoke, not online-serving, FLUX, CUDA, or general backend coverage. ^[PR #5544] +## Qwen-Image nightly coverage and step-execution perf sharing + +- The Qwen-Image online-serving expansion keeps nine feature pytest IDs, but assigns one + checkpoint to each case: Qwen-Image covers CPU offload, TeaCache, Ulysses degree 2, + CFG parallel size 2, and HSDP shard size 2; Qwen-Image-2512 covers step execution, + layerwise offload, Ring degree 2, and VAE patch parallel size 2. Single-card versus + two-card H100 marks remain attached to those cases. This is a nightly coverage matrix, + not evidence that either checkpoint is interchangeable for runtime behavior or numeric + output. +- The Qwen-Image step-execution perf config uses one H100 server with profiler, + `step-execution`, and `max-num-seqs: 8` for sequential 512x512/20-step and + 1536x1536/35-step cases plus the 512x512/20-step concurrency 1/2/4/8 sweep. Keep the + three `benchmark_params` under that shared server and retain their existing H100 baseline + artifacts; the JSON change itself does not establish a performance result or an active + regression threshold. ^[PR #6613] + ## 目录内容 | 遇到什么 | 查看哪里 | diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 9cb7d22ea..b462d412a 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -136,6 +136,14 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 开始；level-2 terminal case 之后不再复用 server；统计目标 lane 确实只初始化预期数量的 engine，且 cleanup 后无 worker/device state 遗留。^[PR #5713] +- Qwen-Image nightly 为减少 cold start 可将 18 个「两 checkpoint × 九 feature」case 收敛为 + 九个「每 feature 一个 checkpoint」case，但必须保留全部 feature pytest ID、原单卡/双卡 H100 + marker 和明确的 checkpoint→feature 映射；这只改变权重 coverage 的频率，不能宣称保留了每个 + feature 在每个权重上的组合覆盖。共享 step-execution perf server 时，保持 sequential 与 + concurrency-sweep `benchmark_params`、共同的 `max-num-seqs: 8` capacity 及既有 H100 + baseline artifact；没有实际 lane 结果或显式 consumer 时，不得把 PR 预期或 JSON baseline + 说成性能不变或 regression-gate verdict。^[PR #6613] + ## OMNI-CI-2d — diffusion tiny builder 必须替代资源缩减而不是能力缩减 - 触发：重模型 online E2E 因显存 OOM 迁移到 tiny-model framework，或修改 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d03a11edc..d33994834 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0d5d1115` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ dd527b42` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 766b549dc..72ee5b4ea 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0d5d1115` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ dd527b42` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2d6367e09..5e5616b42 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0d5d1115` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ dd527b42` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8694a1a29..c2eb8d51e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0d5d1115` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ dd527b42` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8e73c570c..8591ef34f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0d5d1115` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ dd527b42` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 24017ed67..c48754dfc 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0d5d1115` 验证存在 +- 源码校验：以上路径均已在 `main @ dd527b42` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 86a73ed00..189e0bc07 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0d5d1115`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ dd527b42`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From a45132defc085fd5c63534209853332a30ddc58a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:03:25 +0800 Subject: [PATCH 301/465] knowledge(vllm-omni): sync 5a8b81eb (PR #6526) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../models/minimax-h3/rules-deployment.md | 23 ++++++++++++++----- .../vllm-omni/models/minimax-h3/rules.md | 4 ++-- 11 files changed, 29 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ec74b1174..4c9b87e72 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0d5d1115d6f0b8be86b7b9f003705b6698540887 + previous_audited_sha: dd527b42456ad29d5df4fdbfa56040b3c196a75f audited_ref: main - audited_sha: dd527b42456ad29d5df4fdbfa56040b3c196a75f + audited_sha: 5a8b81ebebd28b5bc23a8a0721d39998496e7015 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4db550280..77a14fbc7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -dd527b42456ad29d5df4fdbfa56040b3c196a75f`. The canonical machine baseline +5a8b81ebebd28b5bc23a8a0721d39998496e7015`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d33994834..79e59c602 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ dd527b42` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5a8b81eb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 72ee5b4ea..ecbbad9a1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ dd527b42` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5a8b81eb` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5e5616b42..d2971ec7c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ dd527b42` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5a8b81eb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c2eb8d51e..5c26b4181 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ dd527b42` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5a8b81eb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8591ef34f..550a9721d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ dd527b42` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5a8b81eb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c48754dfc..4ec2e0da7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ dd527b42` 验证存在 +- 源码校验：以上路径均已在 `main @ 5a8b81eb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 189e0bc07..2a3ec4c48 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ dd527b42`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5a8b81eb`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index f22e48eb0..bf1ddb0ed 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py", "PR #6072"] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py", "PR #6072", "PR #6526", tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/offloader/test_module_residency.py] confidence: high --- @@ -21,16 +21,28 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule H2D streaming 使用它；`_supports_mmap_loading=False` 是安全门，不能绕过。plan 将 `token_refiner.blocks` 独立 stream，text encoder 的 vision/text block rank-local stream， video/audio VAE 按 encode/decode stage on-demand，leading `transformer` blocks 只在 denoise - stage resident 并在 decode 前释放。`dlo_resident_layers>0` 必须与 no-AllGather 配对。 + stage resident 并在 decode 前释放。`dlo_resident_layers>0` 必须与 no-AllGather 配对。MiniMax-H3 + 的 staged encoder→DiT→VAE 边界可共用一个 component-local `BoundedAllocatorCache`：仅当 + cached-but-unallocated 不超过设备容量 25% 且物理 free 不低于 5% 时保留；遥测缺失、任何 + component/stager failure 或 OOM retry 都保守释放。encoder 的非 block children 必须作为一个 + immutable pinned-CPU snapshot group，以一条 copy stream/event stage，保持 parameter/buffer + storage alias、shape、stride、offset 与 dtype；DiT prefetch 和两个 shared device buffer 不变。 - 禁止：让 encoder TP shard 进入 DiT AllGather；把 DLO+CPU offload 的早期分支优先级交给 generic CPU-offload hook；只因 component 有 `offload_to_cpu` 就推断每个 caller 都正确 staging。 目标 pin 的 `_encode_audio_conditions(standalone_audios)` 没有进入 `_component_on_device`， 会在前一个 embedded-audio context 已关闭后直接使用 CPU-bound audio VAE；没有外层 stage - context 或专属测试证明 image+standalone-audio Ref2VA 的 DLO device/performance 合同。 + context 或专属测试证明 image+standalone-audio Ref2VA 的 DLO device/performance 合同。不得把此 + component-local retention 变成 global `empty_cache` override，或删除 executor 的 unconditional + shutdown cleanup；不得由 CUDA/no-AllGather 的有界 A/B 外推 AllGather、其他平台、其他 DLO 模型 + 或吞吐/并发收益。 - 验收：分别覆盖 text、image、video、embedded audio、standalone audio 与 decode 的 load→use→finally-offload 次序，异常也释放；regular-loader transform sentinel 到 TP-local stream - 后仍正确；resident layers 只围住 denoise。补齐上述 standalone-audio gap 前，不能用其他 - component staging 单测宣称所有 Ref2VA 输入已覆盖。^[PR #5764] + 后仍正确；resident layers 只围住 denoise。还须覆盖正常保留与两条 memory bound、遥测 fallback、 + OOM 一次 flush/retry、load/offload/component failure 的 host-master restore/forced release，以及 + alias-preserving non-block group 的 idempotent transitions。补齐上述 standalone-audio gap 前，不能用 + 其他 component staging 单测宣称所有 Ref2VA 输入已覆盖。PR #6526 的 L20X matched + single-request no-AllGather CUDA A/B 是有界 latency observation，B300 仅独立 functional smoke； + 二者均非持续 benchmark 或跨 topology/platform performance proof。^[PR #5764] ^[PR #6526] ## MMH3-3b — consumer-GPU profile 是有边界的容量证据 @@ -235,4 +247,3 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule - 强制：`_encode_visual_conditions` 必须在同一个 `video_vae` residency window 内完成该请求的全部 image 与 video encode；`_encode_reference_audio_conditions` 必须在同一个 `audio_vae` residency window 内完成 embedded 与 standalone audio encode。resident helper 只能在对应 scope 内调用，scope 退出时必须释放 component，异常也必须清理。 - 禁止：按 image、video 或 audio 条目分别建立重复的 H2D/D2H lifecycle；在前一个 scope 结束后直接调用 CPU-bound VAE；把单次 image 的行为外推为混合 reference，或把 transfer-level A/B 结果写成完整模型峰值显存与通用性能保证。 - 验收：contract tests 覆盖单/多 image、image+video、embedded+standalone audio 以及 encode exception，精确断言每个 VAE 只有一次 activate/offload；另验证 H3 model-level offload 注册所有实际 DiT 与 VAE stage，并以固定硬件和 workload 单独复核真实性能与输出质量。^[PR #6072] - diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index edb49f908..4592c599e 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526"] confidence: high --- @@ -34,7 +34,7 @@ confidence: high | distilled、DMD2、`base_schedule`、4-step、sigma boundary | [MMH3-2h](rules-cache-task.md#mmh3-2h-distilled-sigma-schedule-按-partition-所有且以区间计步) | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | | LightX2V Turbo、legacy dynamic LoRA、packed QKV/FC1、five sigma points | [MMH3-2j](rules-cache-task.md#mmh3-2j-turbo-lora-只接受精确发布-artifact并与-h3-tasksampling-和-offload-lifecycle-共同门禁) | `lora.py::load_minimax_h3_turbo_lora` → pipeline hook/sampling validation → `DiffusionLoRAManager` | | step execution、continuous batching、multi-document packing、rank-0 prepare failure | [MMH3-2k](rules-cache-task.md#mmh3-2k-h3-step-execution-必须保持请求状态attention-文档和-rank-0-prepare-隔离) | `pipeline_minimax_h3.py::{prepare_encode,denoise_step,step_scheduler}` → `batched_packing.py` → runner | -| DLO、TP-local、resident layers、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → shared DLO backend → pipeline encode/denoise/decode stage contexts | +| DLO、TP-local、resident layers、allocator cache、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → `BoundedAllocatorCache`/`PinnedModuleStager` → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | | 4×H100、DFX perf、T2V/TI2V/V2V、synthetic H.264 reference | [MMH3-3c](rules-deployment.md#mmh3-3c-h100-dfx-fixture-只证明-exact-nightly-workload-与-payload-path) | nightly lane → perf JSON → benchmark request encoder/result artifact | | ROCm、gfx942/gfx950、AITER、BF16、support matrix | [MMH3-3d](rules-deployment.md#mmh3-3d-rocm-support-必须按-sku镜像拓扑和测量协议限界) | support table/footnote → recipe protocol → ROCm backend gate → exact hardware evidence | From 4254fe5105359bdb8c8c537825bd75f960c09ce4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:11:15 +0800 Subject: [PATCH 302/465] knowledge(vllm-omni): sync c608910d (PR #6591) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 4 +-- .../diffusion/rules-checkpoint-loading.md | 15 +++++++-- .../diffusion/rules-component-lifecycle.md | 10 ++++-- .../components/distributed/_index.md | 2 +- .../components/host-weight-runtime/_index.md | 5 +-- .../components/host-weight-runtime/rules.md | 31 ++++++++++++++++++- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 13 files changed, 64 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4c9b87e72..e571cce65 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: dd527b42456ad29d5df4fdbfa56040b3c196a75f + previous_audited_sha: 5a8b81ebebd28b5bc23a8a0721d39998496e7015 audited_ref: main - audited_sha: 5a8b81ebebd28b5bc23a8a0721d39998496e7015 + audited_sha: c608910d20bdd9d56a7570d47cbd52c4a7777395 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 77a14fbc7..aa68c704c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5a8b81ebebd28b5bc23a8a0721d39998496e7015`. The canonical machine baseline +c608910d20bdd9d56a7570d47cbd52c4a7777395`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 79e59c602..b6089af7c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5a8b81eb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c608910d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ecbbad9a1..f1062013c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5a8b81eb` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c608910d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -47,4 +47,4 @@ sources: [] | Cache-DiT、TeaCache 和 prefix cache | [cache acceleration](cache-acceleration.md) | | TP/PP/SP/CFG/VAE/HSDP 等并行策略 | [parallelism](parallelism.md) | | 实验性 world-model session 生命周期、LRU 与内存统计边界 | [session state](session-state.md) | -| checkpoint remap、HSDP/FSDP、final-layout HWR、component quantization 与在线量化加载 | [checkpoint 与加载合同](rules-checkpoint-loading.md) | +| checkpoint remap、HSDP/FSDP、final-layout HWR、registered HWR mmap DLO transport、component quantization 与在线量化加载 | [checkpoint 与加载合同](rules-checkpoint-loading.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index 4385969f6..cd1c9283c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6234", "PR #6279", "PR #6445", "PR #6486", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #4061"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #4061"] confidence: high --- @@ -78,6 +78,9 @@ confidence: high 位置的内容寻址仓库，才可用 blob 名代替内容哈希，其他本地文件和 symlink 必须按内容定身份。 - 禁止：因 symlink 目标名看似 40/64 位十六进制就假设不可变；把 `bf16`、具体 layout 或首个 producer 名写进共享 identity/restorer API；用进程内 inode/mtime guard 代替跨启动身份校验。 +- 强制：node-local canonical-source digest cache 只能是 metadata-validated 的哈希工作复用；路径、 + inode、size、timestamp、symlink target 或 cache-record checksum 任一不符即重算，cache I/O/coordination + 失败也必须直接哈希 canonical source。它不改变 identity 内容、artifact selection 或 fallback policy。 - 验收：第二种 synthetic representation 复用共享 identity/restorer；任意本地十六进制命名 symlink 在同大小内容被替换后产生不同 identity；合法 Hub snapshot→blob 拓扑仍走受控快捷路径。 ^[PR #6445] @@ -191,9 +194,17 @@ confidence: high ordinary DiT materialization，plan/commit 后把 exact final-layout tensors 作为 rank-local DLO source；preferred miss 才 canonical load 并 post-load publish，required 对 miss 或 incompatible artifact fail startup，不能借 producer bootstrap。 +- 强制：eligible warm HWR hit 的 registered transport 只在 `pin_cpu_memory` 开启时尝试，先对完整 + page-aligned mapped ranges 做 read-only capability 与可选 per-worker budget preflight；成功后直接从 + immutable final-layout mmap copy 到既有 HBM buffers，且不分配 private staging。unsupported、budget + exceeded 或已完整 rollback 的 registration failure 必须回到既有两槽 bounded staging；AllGather、 + checkpoint mmap 和全部 zero-interaction gates 不得尝试 registration。该路径不减少 H2D payload， + 也不是 GPU 直接访问 host memory。 - 禁止：把 preferred 当成对 identity/configuration/compatibility error 的宽泛 fallback；让 HWR warm restore 重走 ordinary DiT loader、checkpoint mmap 或 byte-changing finalization；把 no-AllGather 的 rank-local staging 扩展成跨 rank collective 或 generic HWR support。 - 验收：覆盖所有 zero-interaction gates、preferred hit/miss、required miss、mixed/dedicated source rejection、warm path zero ordinary DiT materialization，以及 shared finalization 前后 - restored tensor byte/backing-pointer equality；验证 checkpoint mmap control path 不变。^[PR #6486] + restored tensor byte/backing-pointer equality；验证 checkpoint mmap control path 不变。另覆盖 registration + capability/budget/pin-memory gates、partial registration rollback、direct/staged transfer parity、没有 + staging allocation 的成功路径，以及 teardown 前 unregister。^[PR #6486] ^[PR #6591] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md index 534a96ad2..70472ce2c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5720", "PR #5853", "PR #5884", "PR #6486", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/offloader/startup.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py", "PR #6072"] +sources: ["PR #5720", "PR #5853", "PR #5884", "PR #6486", "PR #6591", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/offloader/startup.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py", "PR #6072"] confidence: high --- @@ -102,9 +102,13 @@ residency 留在模型 owner。规则入口与其他共享机制仍见 [Diffusio enable boundary；该 boundary 必须 take-and-remove state exactly once。backend enable 从 hook/ staging/lease acquisition 到 initial prefetch 失败时先 quiesce and disable，再关闭仍属 loader 的 state；已提交 final-layout restore 在 preferred mode 只能以 fresh canonical model retry，required - mode 必须传播失败。 + mode 必须传播失败。worker shutdown 必须在 destroy distributed state 前 disable offloader；registered + HWR mmap teardown 先 drain/release source references、unregister all ranges，再 close lease。若 unregister + 失败，保留 registration 与 open lease（跨 fresh retry/backend GC 亦然）供 retry 或 process exit，不能 + 让 finalizer unmap CUDA 仍持有的页面。 - 禁止：让 runner 解释 HWR policy 或重复 transfer plan；复用 commit 后的 disposable model；在 async work、partial hooks 或 staging 仍存活时释放 lease，或让 retry 再次 lookup/publish/mmap。 - 验收：覆盖 absent/taken startup state、backend construction/enable/prefetch failure、single fresh retry 与 required no-retry；断言 partial backend disable、carrier/lease close、fresh path bypasses - HWR and checkpoint mmap。^[PR #6486] + HWR and checkpoint mmap；另注入 unregistration failure，断言 retryable retention、backend drop 和 + distributed teardown ordering。^[PR #6486] ^[PR #6591] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d2971ec7c..dc70bfe1d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5a8b81eb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c608910d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md index b2fbb5deb..a073ac25d 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: index tags: [vllm-omni, components] -sources: ["PR #6419", "PR #6486", vllm_omni/host_weight_runtime/] +sources: ["PR #6419", "PR #6486", "PR #6591", vllm_omni/host_weight_runtime/] confidence: high --- @@ -12,7 +12,8 @@ confidence: high - 源码入口：`vllm_omni/host_weight_runtime/`。 - 主要职责：跨模型共享的 immutable host artifact、lease、filesystem store、typed outcome - 和 restore transaction；具体 diffusion producer/identity adapter 仍归 Diffusion owner。 + 和 restore transaction；transport 可在 lease 生命周期内注册 immutable mapping，但具体 diffusion + producer、identity adapter 与 DLO selection 仍归 Diffusion owner。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md index 8b345971f..2dec6f03c 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, components] -sources: ["PR #6419", "PR #6445", "PR #6486", vllm_omni/host_weight_runtime/, "PR #6427"] +sources: ["PR #6419", "PR #6445", "PR #6486", "PR #6591", vllm_omni/host_weight_runtime/, "PR #6427"] confidence: high --- @@ -18,6 +18,7 @@ confidence: high |---|---|---| | typed outcome、preferred/required、producer、timeout | `HWR-1a` | `runtime.py` → `outcomes.py`/`errors.py` → store/writer | | lease/lock、fork、deny、cleanup、filesystem locality | `HWR-1b` | `lease.py` → `filesystem/{locks,store,writer}.py` | +| ordered producer、write-time digest、fsync/READY ordering | `HWR-1d` | `protocols.py` → `filesystem/writer.py` | | plan/commit、manifest/schema、fallback model | `HWR-2a` | `protocols.py`/`manifest.py` → consumer restorer | ## HWR-1a — fallback 只消费显式 retryable typed outcome @@ -48,6 +49,19 @@ confidence: high - 禁止：让 `POST_LOAD_ONLY` producer 进入 pre-load `resolve()`；绕过 runtime 直接调用 store；恢复、rebind 或修改当前启动使用的 canonical model；让 post-load failure 改写已完成的 canonical-fallback resolution。 - 验收：验证 policy-disabled/runtime-disabled 不运行 producer，pre-load 不调用 post-load producer，成功 publication 关闭 lease 且后续 `resolve()` 命中；验证 publication failure、`JOINED`、unexpected store status 和 observer 回调均产生正确的独立 typed report。 ^[PR #6427] +## HWR-1d — publication 优化不得越过 digest 与 durability barrier + +- 触发：producer 增加 `ordered` 声明，或修改 payload write、checksum、`fsync`、manifest/`READY` + publication 顺序。 +- 强制：只有按 canonical storage-key order 发出 payload 的 producer 才可声明 `ordered=True`，由 writer + 在写入时计算 file/tensor SHA256，并将已关闭 payload 的 `fsync` 与后续 producer work 重叠；未声明 + ordered 的 producer 必须保留并行 readback checksum。两条路径都须等待全部 payload digest 与 + `fsync` 完成后才能发布 manifest 和 `READY`。 +- 禁止：用 write-time hash 跳过 durable completion；在 payload future 未完成时暴露 ready artifact； + 把 producer 顺序提示写进 artifact identity，或让 unordered output 复用 ordered fast path。 +- 验收：覆盖 ordered write-time digest、unordered parallel readback、payload `fsync` failure、digest + mismatch 与 publication barrier，断言失败不产生可命中的 `READY` artifact。^[PR #6591] + ## HWR-2a — restore 是 validation-only plan 加一次性 commit - 触发：manifest/schema、producer/restorer identity 或模型 hydration 变化。 @@ -72,3 +86,18 @@ confidence: high - 验收：验证 duplicate take、serialization、loader-side abort、backend-side teardown、enable failure cleanup 与 closed lease rejection；filesystem test 覆盖 writable staging → atomic rename → read-only artifact，以及 hardening failure quarantine。^[PR #6486] + +## HWR-2c — registered mapping 的 ownership 必须晚于 CUDA、早于 lease close + +- 触发：为 immutable HWR mmap 增加/修改 CUDA 或其他 platform host registration、direct H2D、 + registration budget 或 teardown。 +- 强制：transport owner 对完整 page-aligned mapping 先做 capability/budget preflight，再原子地注册； + partial failure 必须 reverse rollback。clean rollback 是 transport unavailable，不改变 HWR identity 或 + artifact outcome；rollback/unregister 留有 live range 则保留 registration 和 open lease 以便 retry， + 且跨 backend GC/fresh retry 仍保持强引用。正常 teardown 必须先停止使用这些 source、unregister every + range，最后 close lease。 +- 禁止：为 registered/staged transport 生成不同 artifact identity；budget/capability miss 后部分启用 + direct H2D；在 CUDA 仍注册时 close/finalize lease；将 clean registration failure 升级成 required-HWR + artifact miss。 +- 验收：覆盖 capability/budget gate、mid-registration rollback、PyTorch pinned-source verification、 + unregister retry 与 retained-owner GC，并断言 successful direct path 无 private staging allocation。^[PR #6591] diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5c26b4181..7523c92ef 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5a8b81eb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c608910d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 550a9721d..206f2b4fc 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5a8b81eb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c608910d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4ec2e0da7..7003c07a1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5a8b81eb` 验证存在 +- 源码校验：以上路径均已在 `main @ c608910d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2a3ec4c48..c529011e4 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5a8b81eb`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c608910d`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 214441b2d5a165f8468fb68ff61de51357794b32 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:16:58 +0800 Subject: [PATCH 303/465] knowledge(vllm-omni): sync 497c537c (PR #6522) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/benchmark/_index.md | 5 +- .../vllm-omni/benchmark/guides/_index.md | 5 +- .../benchmark/guides/omniinteract-realtime.md | 70 +++++++++++++++++++ .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 12 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e571cce65..68a88e7cf 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5a8b81ebebd28b5bc23a8a0721d39998496e7015 + previous_audited_sha: c608910d20bdd9d56a7570d47cbd52c4a7777395 audited_ref: main - audited_sha: c608910d20bdd9d56a7570d47cbd52c4a7777395 + audited_sha: 497c537c6f70e44f376b491bf7b50395cf2cba5d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index aa68c704c..572a67c81 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c608910d20bdd9d56a7570d47cbd52c4a7777395`. The canonical machine baseline +497c537c6f70e44f376b491bf7b50395cf2cba5d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/benchmark/_index.md b/knowledge/repos/vllm-omni/benchmark/_index.md index 4281b803b..05c1a7cd1 100644 --- a/knowledge/repos/vllm-omni/benchmark/_index.md +++ b/knowledge/repos/vllm-omni/benchmark/_index.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 性能与 Profiling" created: 2026-07-10 -updated: 2026-07-10 +updated: 2026-09-04 type: index tags: [vllm-omni, benchmark] -sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml] +sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml, "PR #6522", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py] --- # vLLM-Omni 性能与 Profiling @@ -25,6 +25,7 @@ sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml] | 运行 AR graph 或 Hunyuan benchmark | [benchmark guides](guides/_index.md) | | 调查 profiling、模型加载和性能验证错误 | [benchmark incidents](incidents/_index.md) | | 查询历史结果 | [results](results/_index.md) | +| 运行/审查 OmniInteract native-duplex serving benchmark | [OmniInteract realtime guide](guides/omniinteract-realtime.md) | `benchmarks/tts/model_configs.yaml` 是共享 TTS runner registry；新模型在这里声明 model、deploy、voice/ reference 与 workload，结果保留解析配置、seed、speed、并发和音频属性。IndexTTS 2.5 条目只证明 diff --git a/knowledge/repos/vllm-omni/benchmark/guides/_index.md b/knowledge/repos/vllm-omni/benchmark/guides/_index.md index e5f38dda7..e11f3cf82 100644 --- a/knowledge/repos/vllm-omni/benchmark/guides/_index.md +++ b/knowledge/repos/vllm-omni/benchmark/guides/_index.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 性能指南" created: 2026-07-10 -updated: 2026-07-10 +updated: 2026-09-04 type: index tags: [vllm-omni, benchmark] -sources: [] +sources: ["PR #6522", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py] --- # vLLM-Omni 性能指南 @@ -23,3 +23,4 @@ sources: [] | 验证动态或连续 batching | [dynamic batching](dynamic-batching-validation.md) | | 验证分叉执行路径 | [divergent path](divergent-path-validation.md) | | 查看旧 plan/validation 总入口 | [plan and validation](plan-and-validation.md) | +| 复现或审查 OmniInteract realtime serving runner | [OmniInteract realtime](omniinteract-realtime.md) | diff --git a/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md b/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md new file mode 100644 index 000000000..8ee9b7bb2 --- /dev/null +++ b/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md @@ -0,0 +1,70 @@ +--- +title: "OmniInteract Realtime Benchmark" +created: 2026-09-04 +updated: 2026-09-04 +type: guide +tags: [vllm-omni, benchmark] +sources: ["PR #6522", docs/cli/bench/serve.md, vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/entrypoints/cli/benchmark/cli_args.py] +confidence: high +--- + +# OmniInteract Realtime Benchmark + +`omniinteract` is a `vllm bench serve --omni` dataset: each selected official +`1q1a`、`1q1a_math` or `1qna` video is one MiniCPM-o native-duplex WebSocket +session, not a text-prompt workload. It is a runner/artifact integration; it +checks transport, lifecycle and artifact completeness, not answer accuracy or a +production deployment contract. ^[PR #6522] + +## Invocation and selection contract + +- It requires `--backend openai-realtime-duplex`, `/v1/realtime`, and an + existing `--omniinteract-ref-audio`; incompatible backend/endpoint and + `--ignore-eos`、`--profile`、`--skip-tokenizer-init` or positive + `--probe-request-rate` fail before the run. Omitted endpoint is normalized to + `/v1/realtime` by parser explicitness, not by trusting the upstream default. +- `--num-prompts` omitted means 3; explicit `0` selects all, and an oversize + value selects all available cases. The default concurrency is 1 and any + explicit value must be positive. Dataset input may be an extracted tree, + `data.tar[.gz]`, or a Hugging Face dataset ID; absent path resolves the + official `lucky-lance/OmniInteract` archive. +- Selection is deterministic for seed/`--disable-shuffle`; it validates subset + layout, confined mapping paths, duplicate videos and archive members. + +## Workload, timing and memory boundary + +- Before benchmark timing, every selected video is probed and rejected above + the configured duration limit, then decoded by timeout-bounded `ffmpeg` into + mono 16-kHz PCM16 and at 1 FPS. Input audio replays in 200-ms chunks and + video uses real-time pacing. Thus selected decoded media stays client-resident + and `--max-concurrency` does not bound preparation memory. +- The runner configures the native-duplex session and streams the reference WAV, + input media, final commit, response lifecycle, drained playback ACK and + explicit close. A successful final input needs an accepted model LISTEN or a + non-cancelled response after commit; buffering/unmarked LISTEN and unrelated + pre-commit active-response terminals do not satisfy it. LISTEN-only is valid + benchmark output unless `--omniinteract-require-response` requests functional + E2E semantics. +- Client request timing begins at receipt of `response.created`: TTFT is first + non-empty text delta, TTFP first audio packet, and RTF is through final audio + packet divided by emitted-audio duration. TPOT/ITL derive only from stage-0 + timing; ITL is reported only when all required intervals exist. Missing + measurements remain missing rather than becoming zero, so request goodput + cannot pair metrics from different responses. + +## Artifact and evidence contract + +- A completed case writes `output.wav`, `wav_transcript.json`, `events.json`, + `result.json`, then `.done`; failures use `.failed.json`. The batch writes + `batch_summary.json` and `official_eval_manifest.jsonl`. One output-root lock + serializes complete runs; publication happens after the measured request clock + freezes, so artifact I/O is not serving throughput. +- Transcript timestamps are the serialized playback-queue timeline; raw event + timing remains in `events.json`. Cancelled output or audio clipped beyond the + rounded video horizon stays auditable but is excluded from the official + manifest. Artifact success and official-evaluation eligibility are distinct. +- PR #6522 reported a three-case MiniCPM-o smoke (one case per layout) and + runner tests. Those numbers are environment-specific evidence only: this + commit adds no performance threshold consumer, no answer-quality evaluation, + and no 12-video nightly/production configuration. Do not promote its example + command, default model, or smoke result to a runtime guarantee. diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b6089af7c..3c03a96f3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c608910d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 497c537c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f1062013c..a31adbed7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c608910d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 497c537c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index dc70bfe1d..16f774d12 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c608910d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 497c537c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7523c92ef..74e5561e3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c608910d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 497c537c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 206f2b4fc..d14a97978 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c608910d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 497c537c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7003c07a1..87d904a4e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c608910d` 验证存在 +- 源码校验：以上路径均已在 `main @ 497c537c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c529011e4..7356376c5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c608910d`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 497c537c`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 2c3a3c63330f010e0793309b56e3b61140e13435 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:23:17 +0800 Subject: [PATCH 304/465] knowledge(vllm-omni): sync 0eb47b76 (PR #6598) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-request-input.md | 12 ++++++++++-- .../repos/vllm-omni/components/serving/rules.md | 3 ++- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 22 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 68a88e7cf..64b70fca2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c608910d20bdd9d56a7570d47cbd52c4a7777395 + previous_audited_sha: 497c537c6f70e44f376b491bf7b50395cf2cba5d audited_ref: main - audited_sha: 497c537c6f70e44f376b491bf7b50395cf2cba5d + audited_sha: 0eb47b76d06d563c5d7d59a24865cc92c0dcf08e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 572a67c81..54f048462 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -497c537c6f70e44f376b491bf7b50395cf2cba5d`. The canonical machine baseline +0eb47b76d06d563c5d7d59a24865cc92c0dcf08e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3c03a96f3..cc580d9fe 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 497c537c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0eb47b76` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a31adbed7..779c47439 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 497c537c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 0eb47b76` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 16f774d12..aad7e6b18 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 497c537c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0eb47b76` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 74e5561e3..f922d3219 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 497c537c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0eb47b76` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d14a97978..6a9a2c42e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 497c537c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0eb47b76` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 87d904a4e..55edb2a52 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 497c537c` 验证存在 +- 源码校验：以上路径均已在 `main @ 0eb47b76` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md index e240c2bb0..946fd1d6f 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -4,13 +4,13 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3805", "PR #5374", vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/inputs/, tests/entrypoints/test_omni_entrypoints.py, "PR #5181", "PR #6182"] +sources: ["PR #3805", "PR #5374", "PR #6598", vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/engine/orchestrator.py, vllm_omni/inputs/, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_invalid_audio_speech.py, tests/entrypoints/openai_api/test_serving_speech.py, "PR #5181", "PR #6182"] confidence: high --- # 请求输入合同 -`SERV-4a`–`SERV-4o`：公开请求字段的校验、限界与 owner。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口。 +`SERV-4a`–`SERV-4p`：公开请求字段的校验、限界与 owner。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口。 ## SERV-4a — 公开字段由 serving 显式拥有 @@ -124,3 +124,11 @@ engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)； - 强制：从每个 runtime stage config 取得约束并与同 stage caller params 合并；pipeline-required key 覆盖 caller 冲突值，其他 caller fields 保留。对 mapping 复制合并；对 dataclass/msgspec sampling object 以合并值重建，使 constructor/post-init 重新计算 stop-token 等 derived state，且 caller/default object 保持不变。 - 禁止：caller 提供 params 时整包替换 pipeline constraints；只从 dataclass 而不是实际 OmegaConf runtime config 读取；对已构造 `SamplingParams` setattr/copy 后跳过 derived-state 更新，或静默以 caller 值赢得 pipeline-required field。 - 验收：真实 stage config conversion 覆盖单与多 stage、normal caller field、pipeline-required `detokenize`/stop token conflict、derived stop-token state 及 caller immutability；默认 params 与无约束 control 保持既有行为。^[PR #6182] + +## SERV-4p — 可序列化整数必须在协议层限界且 dispatch 保底 + +- 触发：公开 OpenAI 请求模型新增/修改整数参数，或请求会跨 stage 以 msgpack 传输的 sampling、extra 或嵌套参数。 +- 强制：能在协议模型明确归属的整数必须以有符号 64 位范围校验；计数/尺寸/steps/token fields 保留既有正数或非负下界，seed 可使用完整 `[-2**63, 2**63-1]` 范围。协议拒绝应保留 Pydantic 的字段定位 validation 4xx。所有 stage dispatch 边界仍须将 payload 的 `OverflowError` 转为仅该 request 的非 fatal 400，并经共享 cleanup abort 全部 stage/CFG 状态，orchestrator 继续服务。 +- 禁止：只靠 protocol allowlist 防御嵌套或 extra 参数；把 overflow 作为 engine-wide fatal；捕获无关异常来掩盖程序错误；为统一范围而丢失原有业务下界。 +- 验收：覆盖明确字段的越界 validation（含 speech token/seed 边界）和 mock dispatch 序列化 overflow；后者断言 400、request state 已清理、thread 仍存活。每个修改的公开整数字段须覆盖合法边界与非法一侧。^[PR #6598] +- 边界与未合入建议：目标只捕获 `OverflowError`，不证明其他序列化失败的可恢复性。PR merge 后仍有两个未 resolve review threads：三个 protocol 文件重复定义 int64 constants，且负 TTS seed 的用户可见放宽未写入 PR body；前者是去重建议，后者可由 merged code/tests 与提交历史确认，但两者都不是本提交新增的 shared-constant 或 release-note 合同。 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 8a83c7dc9..b61fe4d5e 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -29,6 +29,7 @@ confidence: high | TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | | request-level LoRA、AR-only/multistage、streaming input update、stage cardinality | `request-contract`：`SERV-4l` | `serving_chat.py` LoRA resolve → `AsyncOmni.generate` → `AsyncOmniEngine._build_add_request_message` → stage-0 input processor | | caller sampling params、pipeline `sampling_constraints`、stage config runtime extraction | `request-contract`：`SERV-4o` | `OmniBase.resolve_sampling_params_list` → `_get_sampling_constraints_list` → `_apply_sampling_constraints` | +| OpenAI integer bounds、msgpack overflow、request-local cleanup | `request-contract`：`SERV-4p` | protocol request models → orchestrator stage dispatch → `_fail_request_client_error` | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | | video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | @@ -45,7 +46,7 @@ confidence: high | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | | `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5m`–`SERV-5n`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | -| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints | `SERV-4a`–`4h`, `SERV-4l`–`4o`，全部见 [请求输入合同](rules-request-input.md) | +| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds | `SERV-4a`–`4h`, `SERV-4l`–`4p`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 7356376c5..bf9c1e0f9 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 497c537c`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0eb47b76`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 5a198a5f01583be1145c8d293d98272e51a036f8 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:28:21 +0800 Subject: [PATCH 305/465] knowledge(vllm-omni): sync 6639c82a (PR #6303) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 17 ++++++++++++++++- knowledge/repos/vllm-omni/ci/rules.md | 4 +++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 29 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 64b70fca2..cee8dbb64 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 497c537c6f70e44f376b491bf7b50395cf2cba5d + previous_audited_sha: 0eb47b76d06d563c5d7d59a24865cc92c0dcf08e audited_ref: main - audited_sha: 0eb47b76d06d563c5d7d59a24865cc92c0dcf08e + audited_sha: 6639c82a171c4801895dae658a7feb8a6295f522 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 54f048462..a8cfa639c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0eb47b76d06d563c5d7d59a24865cc92c0dcf08e`. The canonical machine baseline +6639c82a171c4801895dae658a7feb8a6295f522`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 4b0016dba..47f69c7e8 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6613", tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json] --- # vLLM-Omni CI @@ -33,6 +33,21 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie shape, and nonzero variance. It is an offline inference smoke, not online-serving, FLUX, CUDA, or general backend coverage. ^[PR #5544] +## Diffusion GGUF nightly plugin compatibility + +- The L4 full-model diffusion-quantization nightly lane installs + `vllm-gguf-plugin==0.0.4`, rather than the floating `>=0.0.3`: the later resolved + 0.0.5 imports `huggingface_hub.ResolvedRevision`, which the lane's existing + `huggingface_hub` does not export, so plugin loading and `gguf` registration fail + before the tests run. This is a temporary CI-environment reproducibility pin, not a + runtime support or package-dependency claim. Remove it only after a plugin release + is compatible with that environment or declares the required minimum hub version. +- The scoped command is `pytest -sv tests/diffusion/quantization -m 'full_model and + cuda and L4' --run-level full_model`; its two GGUF cases cover Z-Image-Turbo and + FLUX.2-klein. PR evidence reports local loading/registration recovery, while the + L4 full-model run was deferred to Buildkite, so do not record a full test pass from + this change alone. ^[PR #6303 / issue #6022] + ## Qwen-Image nightly coverage and step-execution perf sharing - The Qwen-Image online-serving expansion keeps nine feature pytest IDs, but assigns one diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index b462d412a..43b5455d5 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -90,11 +90,13 @@ collection 意图；没有实际 runtime 结果时，不能证明目标硬件行 - 触发：修改 pre-commit/Buildkite schema、下载 lint 工具、SPDX/import 检查或 stability marker。 - 强制：固定工具版本并校验摘要；声明本地与 CI 强制 hook 矩阵；路径跨 Windows/POSIX 归一化；源码头使用项目 SPDX 标识且覆盖 `.sh`；marker 参数 shell-quote 并有明确默认值。 +- CI 测试插件若与既有环境依赖不兼容，必须在受影响 lane 精确 pin 到已验证版本，并记录失效 + 的解析版本、冲突符号/依赖和解除条件；不能把 lane 的临时 pin 误写成运行时支持或项目依赖合同。 - 禁止：下载浮动 `stable`；让 rebase 静默回退 hook/marker；未引用包含 `: ` 的 YAML command； 用测试名后缀代替硬件 marker。 - 验收：当前 main schema/pre-commit、DCO 和 marker collection 通过；篡改摘要、Windows 路径、 shell header 或非法 stability 参数分别被对应 gate 拒绝。 ^[PR #3422] ^[PR #6273] - ^[PR #6293] ^[PR #6343] + ^[PR #6293] ^[PR #6303] ^[PR #6343] - pre-commit ratchet 不能把债务下降当成 hook 失败：`actual > budget` 才返回 1， 相等静默通过， `actual < budget` 必须返回 0 并打印收紧上限的 reminder。passing hook 需 `verbose: true` diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index cc580d9fe..9c0514ce9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0eb47b76` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6639c82a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 779c47439..b38272d3c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0eb47b76` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6639c82a` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index aad7e6b18..9bb631936 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0eb47b76` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6639c82a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f922d3219..b206c161a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0eb47b76` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6639c82a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6a9a2c42e..0e6dba328 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0eb47b76` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6639c82a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 55edb2a52..6ff4001a8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0eb47b76` 验证存在 +- 源码校验：以上路径均已在 `main @ 6639c82a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index bf9c1e0f9..d97740476 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0eb47b76`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6639c82a`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From aa17c157f013eadd324080eb078f6538f12f6e70 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:33:18 +0800 Subject: [PATCH 306/465] knowledge(vllm-omni): sync 780e8d40 (PR #6217) --- adapters/vllm_omni/release_baseline.yaml | 5 ++- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 3 +- .../environment-variable-contract.md | 37 +++++++++++++++++++ .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 10 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cee8dbb64..ab28250f6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0eb47b76d06d563c5d7d59a24865cc92c0dcf08e + previous_audited_sha: 6639c82a171c4801895dae658a7feb8a6295f522 audited_ref: main - audited_sha: 6639c82a171c4801895dae658a7feb8a6295f522 + audited_sha: 780e8d40aa01cddb686472e3e60e5c7e6f117a09 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -33,6 +33,7 @@ path_owners: - .github/ - docker/ configuration: + - collect_env.py - vllm_omni/config/ - vllm_omni/deploy/ diffusion: diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a8cfa639c..fd2ad2352 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6639c82a171c4801895dae658a7feb8a6295f522`. The canonical machine baseline +780e8d40aa01cddb686472e3e60e5c7e6f117a09`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9c0514ce9..31a417efd 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6639c82a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 780e8d40` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 @@ -42,6 +42,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] | 查询 `Omni()` 初始化参数 | [Omni init args](omni-init-args.md) | | 写或修改 deploy YAML | [deploy YAML](deploy-yaml.md) | | 核对 pipeline/deploy schema | [pipeline deploy schema](pipeline-deploy-schema.md) | +| 审查或新增 process 环境变量、stage `env` 和安全诊断输出 | [environment-variable contract](environment-variable-contract.md) | | 计算多 stage 显存预算 | [deployment memory budget](deployment-memory-budget.md) | | 添加新模型和注册点 | [adding a model](adding-a-model.md) | | composable strategy 的 axis、并行拓扑与 load balancing owner | [并行拓扑合同](rules-parallel-topology.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md new file mode 100644 index 000000000..8d1da5612 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md @@ -0,0 +1,37 @@ +--- +title: "环境变量合同与诊断边界" +created: 2026-09-04 +updated: 2026-09-04 +type: guide +tags: [vllm-omni, config, environment] +sources: ["PR #6217", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] +--- + +# 环境变量合同与诊断边界 + +## 什么时候查这里 + +- 新增、审查或排查 process 环境变量，尤其是 deploy stage `env`、worker 继承和 `collect_env.py` 输出。 +- 判断一个从源码搜到的名字是否是稳定公开配置，而不是模型、benchmark、平台或内部开关。 + +先从 [Configuration index](_index.md) 选择 owner，再把 env 与 typed deploy 字段的最终值按 +[config audit](config-audit-plain-language.md) 的 producer→consumer 链路核实。 + +## 已合并的合同 + +- `environment_variable_inventory.py` 是**分类清单，不是值解析器**：实际 consumer 继续拥有默认值、解析、优先级和读取时机。仅 `PUBLIC_OMNI` 的 22 个名称构成审阅后的 Omni 公共边界；新公共 Omni 名称必须以 `VLLM_OMNI_` 开头，旧前缀仅为兼容保留。 +- 公开设置没有全局的 env-vs-CLI/YAML/request 优先级。按每个 consumer 的合同核实：请求可变行为应属于 request schema，稳定 model/stage 行为应属于 typed config，环境变量只适合启动、平台集成、诊断或兼容回退。import/startup 时读取的值必须在启动 CLI/导入前设置；启动后的父 shell 变化不会更新既有 worker。 +- stage 的 `env`（以及兼容的 `runtime.env`）仅在启动该 stage/child process 时临时应用：键和值会字符串化，child 可继承，随后恢复父进程旧环境。因此它不能追溯改变已在此前 import 缓存的值，也不应承载 request-varying 语义。日志和诊断只输出键，绝不输出 stage `env` 的值。 +- `collect_env.py` 只输出安全的公共 Omni allowlist、注册的 vLLM 名称和选择的平台前缀；显式 redact 的 `HF_TOKEN`、`HUGGINGFACE_HUB_TOKEN`、`OPENAI_API_KEY` 及常见 secret-like 名称不得输出值。为了诊断不完整/损坏的 Omni 安装，清单导入失败时 collector 回退为空清单而仍可报告既有安全信息。 +- 静态 drift 测试覆盖可解析的直接 `os` 访问、module 常量、local wrapper、别名、membership 与小写 key，并对 model/benchmark transitional 行做反向文本存活检查；Pydantic 生成的 server-storage 名称单独核实。动态拼接的跨模块 metadata 和 stage `env` 任意键仍不能由扫描可靠推断，必须人工分类。 + +## 审查门禁 + +1. 先沿真实 consumer 确认解析、默认、优先级与读取时机；不要把 inventory 或文档表当成运行时实现。 +2. 新的稳定公开 Omni 设置同时更新分类、公开参考覆盖和 prefix gate；模型专用、benchmark、platform/external、internal 名称保持各自 owner，不因被读取而自动公开。 +3. 对 model-specific name 指定迁移目的（typed config、request scope、external、internalize 或 deprecate/remove）但不要把目的当成已经实现。PR #6217 的既有 57 个迁移项由 #6232 按模型族和 owner 跟踪，尚不是现有运行时保证。 +4. 变更 collector 或 stage `env` 时，以真实 worker/inheritance 与 redaction 测试验收；错误安装场景不得因可选清单导入而使诊断工具本身失效。 + +## 已知证据限制 + +PR #6217 的清单与 CI 防止可静态识别的 drift，但不证明所有动态环境访问均已发现，也不统一历史 consumer 的 precedence。清单是该提交的审阅快照；main 并发演进仍可能引入 drift，因此审查当前分支时必须运行 live scanner，并以当前 source 而非本页的历史计数判断。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b38272d3c..328aa8a16 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6639c82a` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 780e8d40` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9bb631936..8a24e57f4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6639c82a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 780e8d40` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b206c161a..72483da51 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6639c82a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 780e8d40` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0e6dba328..72689d3b1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6639c82a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 780e8d40` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6ff4001a8..ad265f3f2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6639c82a` 验证存在 +- 源码校验：以上路径均已在 `main @ 780e8d40` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d97740476..323ea5e6f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6639c82a`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 780e8d40`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From a03cc4b59fcfb07b70a3b736208e2175af058ca0 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:37:23 +0800 Subject: [PATCH 307/465] knowledge(vllm-omni): sync a708ae5b (PR #6631) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../configuration/environment-variable-contract.md | 7 ++++--- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 10 files changed, 14 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ab28250f6..82c0026e8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6639c82a171c4801895dae658a7feb8a6295f522 + previous_audited_sha: 780e8d40aa01cddb686472e3e60e5c7e6f117a09 audited_ref: main - audited_sha: 780e8d40aa01cddb686472e3e60e5c7e6f117a09 + audited_sha: a708ae5b7aac7ccd86f63cdc7cb853d68ede3284 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index fd2ad2352..425d63c5e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -780e8d40aa01cddb686472e3e60e5c7e6f117a09`. The canonical machine baseline +a708ae5b7aac7ccd86f63cdc7cb853d68ede3284`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 31a417efd..94cba7a02 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 780e8d40` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a708ae5b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md index 8d1da5612..9a9c5a725 100644 --- a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md +++ b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: guide tags: [vllm-omni, config, environment] -sources: ["PR #6217", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] +sources: ["PR #6217", "PR #6631", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] --- # 环境变量合同与诊断边界 @@ -19,7 +19,8 @@ sources: ["PR #6217", vllm_omni/config/environment_variable_inventory.py, tests/ ## 已合并的合同 -- `environment_variable_inventory.py` 是**分类清单，不是值解析器**：实际 consumer 继续拥有默认值、解析、优先级和读取时机。仅 `PUBLIC_OMNI` 的 22 个名称构成审阅后的 Omni 公共边界；新公共 Omni 名称必须以 `VLLM_OMNI_` 开头，旧前缀仅为兼容保留。 +- `environment_variable_inventory.py` 是**分类清单，不是值解析器**：实际 consumer 继续拥有默认值、解析、优先级和读取时机。`main @ a708ae5b` 的 `PUBLIC_OMNI` 审阅边界有 23 个名称；新公共 Omni 名称必须以 `VLLM_OMNI_` 开头，旧前缀仅为兼容保留。 +- PR #6631 处理 #6217 合并后的 main 漂移：`VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` 是公开、实验性、仅环境变量的 float 秒数开关，默认 `600`；diffusion engine 在每次 `step_streaming` 请求路径读取它，非 float 或 `<=0` 时仅 warning-once 并回退默认值。它等待 async output，不是 `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` 的 full-payload input-coordinator 控制。`ROSVOT_SOURCE_DIR` 已随 SoulX-Singer 支持移除而从 model-specific `deprecate_remove` 清单删除。 - 公开设置没有全局的 env-vs-CLI/YAML/request 优先级。按每个 consumer 的合同核实：请求可变行为应属于 request schema，稳定 model/stage 行为应属于 typed config，环境变量只适合启动、平台集成、诊断或兼容回退。import/startup 时读取的值必须在启动 CLI/导入前设置；启动后的父 shell 变化不会更新既有 worker。 - stage 的 `env`（以及兼容的 `runtime.env`）仅在启动该 stage/child process 时临时应用：键和值会字符串化，child 可继承，随后恢复父进程旧环境。因此它不能追溯改变已在此前 import 缓存的值，也不应承载 request-varying 语义。日志和诊断只输出键，绝不输出 stage `env` 的值。 - `collect_env.py` 只输出安全的公共 Omni allowlist、注册的 vLLM 名称和选择的平台前缀；显式 redact 的 `HF_TOKEN`、`HUGGINGFACE_HUB_TOKEN`、`OPENAI_API_KEY` 及常见 secret-like 名称不得输出值。为了诊断不完整/损坏的 Omni 安装，清单导入失败时 collector 回退为空清单而仍可报告既有安全信息。 @@ -29,7 +30,7 @@ sources: ["PR #6217", vllm_omni/config/environment_variable_inventory.py, tests/ 1. 先沿真实 consumer 确认解析、默认、优先级与读取时机；不要把 inventory 或文档表当成运行时实现。 2. 新的稳定公开 Omni 设置同时更新分类、公开参考覆盖和 prefix gate；模型专用、benchmark、platform/external、internal 名称保持各自 owner，不因被读取而自动公开。 -3. 对 model-specific name 指定迁移目的（typed config、request scope、external、internalize 或 deprecate/remove）但不要把目的当成已经实现。PR #6217 的既有 57 个迁移项由 #6232 按模型族和 owner 跟踪，尚不是现有运行时保证。 +3. 对 model-specific name 指定迁移目的（typed config、request scope、external、internalize 或 deprecate/remove）但不要把目的当成已经实现。目标提交的 56 个 model-specific 项为 promote 33、request scope 6、external 0、internalize 11、deprecate/remove 6；#6232 仍按模型族和 owner 跟踪迁移，尚不是现有运行时保证。该 issue 的旧 seven-item removal checklist 含已退休的 `ROSVOT_SOURCE_DIR`，不能覆盖当前 inventory。 4. 变更 collector 或 stage `env` 时，以真实 worker/inheritance 与 redaction 测试验收；错误安装场景不得因可选清单导入而使诊断工具本身失效。 ## 已知证据限制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 328aa8a16..45f309d06 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 780e8d40` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ a708ae5b` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8a24e57f4..0146e64e5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 780e8d40` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a708ae5b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 72483da51..5ffe36aaf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 780e8d40` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a708ae5b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 72689d3b1..71fb638eb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 780e8d40` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a708ae5b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ad265f3f2..ac0547190 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 780e8d40` 验证存在 +- 源码校验：以上路径均已在 `main @ a708ae5b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 323ea5e6f..b36600ad5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 780e8d40`（2026-08-25）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a708ae5b`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 8a5c2652dd1dac621f323742b45374cbe9ed6a9a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:43:03 +0800 Subject: [PATCH 308/465] knowledge(vllm-omni): sync 40dcfdbb (PR #6040) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 11 ++++++++++- 10 files changed, 20 insertions(+), 11 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 82c0026e8..5a63668f5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 780e8d40aa01cddb686472e3e60e5c7e6f117a09 + previous_audited_sha: a708ae5b7aac7ccd86f63cdc7cb853d68ede3284 audited_ref: main - audited_sha: a708ae5b7aac7ccd86f63cdc7cb853d68ede3284 + audited_sha: 40dcfdbb4343fca917b34287cba620eb436a856d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 425d63c5e..8885cff58 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -a708ae5b7aac7ccd86f63cdc7cb853d68ede3284`. The canonical machine baseline +40dcfdbb4343fca917b34287cba620eb436a856d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 94cba7a02..24d2079a5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a708ae5b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 40dcfdbb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 45f309d06..51f8bbd57 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a708ae5b` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 40dcfdbb` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0146e64e5..73ade8def 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a708ae5b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 40dcfdbb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5ffe36aaf..314d7f946 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a708ae5b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 40dcfdbb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 71fb638eb..f2ba1c2e2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a708ae5b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 40dcfdbb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ac0547190..582ee0641 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a708ae5b` 验证存在 +- 源码校验：以上路径均已在 `main @ 40dcfdbb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b36600ad5..b306a8297 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a708ae5b`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 40dcfdbb`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 4592c599e..d03a323e4 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", "PR #6040", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526"] confidence: high --- @@ -24,6 +24,7 @@ confidence: high | TRTLLM、ragged packed metadata、SAGE、Skip-Softmax、Blackwell default | `MMH3-1e` | H3 metadata/roles → TRTLLM packed trim/quant gate → platform default | | text encoder、missing q/k/v 或 gate/up、eager load bookkeeping | [MMH3-1f](rules-loading.md#mmh3-1f-text-encoder-eager-load-必须证明每个-source-shard-完整) | `encoder.py::_load_weights` → `pipeline_minimax_h3.py::load_weights` strict report | | NPU packed varlen、quadratic mask、LaserAttention、prefix K/V | `MMH3-1g` | H3 packed producer → backend capability/metadata → NPU FlashAttention fallback | +| Qwen3-VL encoder、NPU causal GQA、AddRMSNorm residual | `MMH3-1k` | encoder SDPA helper/residual call → NPU model patch → `torch_npu` op | | FL2VA keyframe、Ref2VA mixed reference/时域限界、shape/output matrix | [MMH3-2a](rules-media.md#mmh3-2a-taskreferenceshape-与多输出必须作为一个输入矩阵维护) | `pipeline_minimax_h3.py` → `reference_video.py` | | media limit、typed/multipart reference、HTTP 400、temp source | [MMH3-2b](rules-media.md#mmh3-2b-media-ingress-在解码前限界request-错误保持-http-400) | `api_server.py` → `serving_video.py` → `reference_video.py` | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | [MMH3-2c](rules-cache-task.md#mmh3-2c-conditioned-vae-的固定种子必须按实际设备隔离并恢复) | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | @@ -193,6 +194,14 @@ confidence: high - 禁止：把 batch-specific frequency 广播、展平或转置成单样本语义；绕过共享 helper 直接调用 NPU 算子而失去 fallback；用 helper 直测代替生产 platform hook 验证，或把该 text-encoder 优化外推到其他 H3 RoPE 路径和平台。 - 验收：NPU correctness tests 覆盖 BNSD fast shapes、BSND fallback shapes 和不同样本的真实 M-RoPE positions，分别以 NeoX reference 核对 q/k 输出并断言样本频率确实不同；另从平台注册入口验证生产 consumer 已启用 fused path。^[PR #6061] +## MMH3-1k — Qwen3-VL encoder 的 NPU GQA 与 AddRMSNorm 只改变目标平台合同 + +- 触发：修改 Qwen3-VL text encoder 的 `_scaled_dot_product_attention`、post-attention residual、NPU MiniMax-H3 patch 或 `RMSNorm` 的 optional residual 参数。 +- 强制：通用 encoder helper 保持 causal SDPA 的 reference 语义：当本地 Q heads 多于 K/V heads 时先沿 head 维 repeat-interleave K/V；NPU patch 才保留压缩 BNSD `[B,Q,S,D]` / `[B,KV,S,D]` 输入并调用 `F.scaled_dot_product_attention(..., dropout_p=0.0, is_causal=True, enable_gqa=Q != KV)`。进入该 NPU op 前必须拒绝 K/V head 数不等、零 KV head 和不能整除的 Q:KV 比率。patch 与已有 RoPE/SwiGLU patch 独立且幂等，并仅在 NPU diffusion runtime 已建立、pipeline 尚未加载时注册到此 encoder helper。 +- 强制：shared `RMSNorm(x, residual)` 的语义为先计算 `updated_residual = residual + x`，再以该值 norm，返回 `(normalized, updated_residual)`；MiniMax-H3 decoder 必须将此 updated residual 用于后续 MLP residual add。仅 NPU residual 分支调用 `torch_npu.npu_add_rms_norm`；CUDA、HIP、MUSA 和 XPU residual 分支保留 native equivalent，不能由这个补丁推断其他平台获得 fused AddNorm。 +- 禁止：把 NPU compressed-GQA patch 提升为 shared diffusion Attention/backend policy、改变 DiT attention 或其他 encoder，或用 NPU mock wiring 宣称真实 kernel parity。不得把 PR 的单次 A3 eight-card FL2VA observation（one warmup；249028.120→248662.942 ms）写成跨 workload、checkpoint、topology 或平台的性能保证。 +- 验收：CPU contract 覆盖通用 expanded-K/V fallback 与 decoder residual order；NPU 覆盖独立/幂等生产 patch、valid/invalid head shapes，以及真实 BF16 causal GQA 相对 expanded-K/V reference（PR fixture：`B=1,Q=8,KV=2,S=16,D=128`，`atol=rtol=2e-2`）。还应保留 native NPU op 的目标硬件数值测试；mock 仅可证明 parameters and registration wiring。^[PR #6040] + ## MMH3-1j — strict Ulysses 必须保持局部 embedding 与 projection-before-gather 的边界 - 触发：修改 MiniMax-H3 的 strict Ulysses、`_embed()`、packed RoPE、`local_sp_prepare`、DiT final projection 或 `sp_gather` 边界。 From 6cc12ed3d2c1f59a1a05957ac7d5c1c1e23b13e1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 19:49:25 +0800 Subject: [PATCH 309/465] knowledge(vllm-omni): sync aeb348d8 (PR #6357) --- adapters/vllm_omni/release_baseline.yaml | 8 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../docs/generation-and-support-rules.md | 6 ++- knowledge/repos/vllm-omni/models/catalog.md | 18 ++++--- .../vllm-omni/models/dreamid-omni/_index.md | 47 ++++++------------- .../vllm-omni/models/magi-human/_index.md | 47 +++++-------------- 12 files changed, 54 insertions(+), 86 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5a63668f5..081cfcfcb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a708ae5b7aac7ccd86f63cdc7cb853d68ede3284 + previous_audited_sha: 40dcfdbb4343fca917b34287cba620eb436a856d audited_ref: main - audited_sha: 40dcfdbb4343fca917b34287cba620eb436a856d + audited_sha: aeb348d8f8a555f0f9e91bee49f48778744a7bc6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -13,8 +13,8 @@ inventories: count: 81 sha256: 9d5a685eae7ef099bb884557ca51b0e1b5f19c8555aa16bbd136c2081d95cd78 diffusion: - count: 67 - sha256: b11ab867faf8faa8f2b9a81254d178333abffd6e3adcf7c932399fabe162accb + count: 65 + sha256: 6107384ad499e668c1ed3690d740b7be816c900c504f244dce557ef51a257f2a pipelines: count: 55 sha256: 9d5033c9a7ffd7ff636e32db0f9be2a549b7c16b034e13817e280f19a9e16769 diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8885cff58..35697198c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -40dcfdbb4343fca917b34287cba620eb436a856d`. The canonical machine baseline +aeb348d8f8a555f0f9e91bee49f48778744a7bc6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 24d2079a5..8eabd832b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 40dcfdbb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ aeb348d8` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 51f8bbd57..285471f8a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 40dcfdbb` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ aeb348d8` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 73ade8def..07c85d7e3 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 40dcfdbb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ aeb348d8` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 314d7f946..065afaca7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 40dcfdbb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ aeb348d8` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f2ba1c2e2..b69909632 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 40dcfdbb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ aeb348d8` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 582ee0641..56b25cb10 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 40dcfdbb` 验证存在 +- 源码校验：以上路径均已在 `main @ aeb348d8` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md index 059f57477..1219e0c49 100644 --- a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -18,7 +18,7 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, `generate_examples.py` 的扫描/生成顺序，或手改 User Guide → Examples navigation。 - 强制：`offline_inference` 与 `online_serving` 只有固定 shared-task slug 可生成页面并进入 nav： `image_to_image`、`image_to_video`、`speech_to_video`、`text_to_audio`、`text_to_image`、 - `text_to_speech`、`text_to_video`、`x_to_text`、`x_to_video_audio`。过滤必须在 page write 与 nav + `text_to_speech`、`text_to_video`、`x_to_text`。过滤必须在 page write 与 nav update 之前执行；model-specific serving source 继续留在 `examples/`，不能重新与 task 入口混排； 若要从文档侧发现它，必须由相应 shared task page 显式链接，不能只依赖 source directory 存在。 非 serving category 会全部通过 filter，仍按 category/stem 生成 @@ -35,7 +35,9 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, serving slug 不生成/不入 nav且能从相应 shared page 到达，非 serving category 保持生成；再验证 重复 nav rebuild、preserved string 与 source-derived URL。target test 只用 synthetic `text_to_image`/`qwen3_omni` 检查 predicate 两端，没有执行 page generation、nav serialization、 - 完整 whitelist、shared-page link coverage 或真实树 census。 + 完整 whitelist、shared-page link coverage 或真实树 census。`x_to_video_audio` 曾是 DreamID-Omni + 与 MagiHuman 唯一共用的 offline task；PR #6357 同时删除其 source、generated page、nav 项和 + whitelist entry。不得恢复该 slug，除非新的共享 task source、页面、nav 和测试一起落地。^[PR #6357] ^[PR #5987] ### 已知完整性缺口 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b306a8297..517c221e1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 40dcfdbb`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ aeb348d8`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -36,7 +36,7 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | -| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 67 条 pipeline / 42 个模型族目录 | +| Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 40 个模型族目录 | | Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 55 个 key | | Deploy YAML | `vllm_omni/deploy/*.yaml` | 84 份 | @@ -47,6 +47,12 @@ architecture、两个 pipeline key、两个 deploy YAML 及其 tokenizer/config/ 因此 active owner 与 catalog 不再声明这两个家族；diffusion text-to-audio 的 first-party 示例只剩 Stable Audio，Audex 的 TTA 仍是独立两阶段 autoregressive 路径。^[PR #6353] +PR #6357 删除 `DreamIDOmniPipeline` 和 `MagiHumanPipeline`，以及相应的模型目录、MagiHuman +extras、测试、文档和唯一共享的 `x_to_video_audio` offline 示例；这两个家族不再是当前 +first-party 支持，下面的 registry/owner 清单不将其列为 active。两页保留为历史 tombstone， +不得作为可运行支持或能力证据。RFC #5996 最初讨论的是 CI delist；本 PR 的合并代码和 PR +body 明确扩大为完整 first-party 删除。^[PR #6357] ^[Issue #5996] + PR #4820 将 `WanDMDPipeline` 注册为 Wan2.2 的同实现入口，因此 diffusion pipeline 计数回升至 65；它不是新的模型族或新的 `OMNI_PIPELINES` key。^[PR #4820] @@ -63,12 +69,12 @@ mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、minimax_music3、moss_t moss_tts_nano、nemotron_voicechat、omnivoice、personaplex、qwen2_5_omni、qwen3_omni、 qwen3_tts、step_audio2、voxcpm2、voxtral_tts -## Diffusion 模型族（42） +## Diffusion 模型族（40） -bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamid_omni、 -dreamzero、ernie_image、flux、flux2、flux2_klein、glm_image、gr00t、helios、 +bagel、boogu_image、cosmos3、diffusers_adapter（通用 diffusers 桥）、dreamzero、ernie_image、 +flux、flux2、flux2_klein、glm_image、gr00t、helios、 hidream_image、hidream_o1_image、hunyuan_image3、hunyuan_video、internvla_a1、krea2、lance、 -lingbot_video、lingbot_world、longcat_image、longcat_video、ltx2、magi_human、ming_flash_omni、 +lingbot_video、lingbot_world、longcat_image、longcat_video、ltx2、ming_flash_omni、 minimax_h3、nextstep_1_1、omnigen2、omnivoice、ovis_image、pi0、qwen_image、sana_video、sana_wm、sd3、 sdxl、sensenova_u1、stable_audio、wan2_2、z_image diff --git a/knowledge/repos/vllm-omni/models/dreamid-omni/_index.md b/knowledge/repos/vllm-omni/models/dreamid-omni/_index.md index 99a3203ed..4be5cfa22 100644 --- a/knowledge/repos/vllm-omni/models/dreamid-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/dreamid-omni/_index.md @@ -1,41 +1,22 @@ --- -title: "DreamID-Omni（Wan2.2 基座音视频身份生成）" +title: "DreamID-Omni（已移除的 first-party 支持）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/dreamid_omni/, vllm_omni/diffusion/registry.py] +sources: ["PR #6357"] --- -# DreamID-Omni +# DreamID-Omni — historical tombstone -以下事实在 `main @ 5d44868e` 复核（源码派生页,尚无本模型的运行经验沉淀）。 +PR #6357 removed first-party DreamID-Omni support at `main @ aeb348d8`: its +`DreamIDOmniPipeline` registry entry and pre/post-processing registration, the +`vllm_omni/diffusion/models/dreamid_omni/` implementation, model tests, setup +script, documentation, and the shared `x_to_video_audio` example are gone. -## 名称与范围 - -- 无别名、无变体有据,树内未 pin checkpoint。diffusion registry: - `DreamIDOmniPipeline` → - （`dreamid_omni`, `pipeline_dreamid_omni`, `DreamIDOmniPipeline`）,post - `get_dreamid_omni_post_process_func`。单 stage diffusion,引擎默认 stage - 配置（[Config 组件](../../components/configuration/architecture.md)）。无 deploy YAML。 -- 联合音频+视频身份保持生成:家族目录内含 TP 优化的 Wan2.2 相关实现 - （`wan2_2.py`：`WanSelfAttention` + 跨 TP rank 求全局 RMS 的 - `DistributedRMSNorm`）,但整体**硬依赖树外 `dreamid_omni` 包**（见下）; - Wan 家族本体见 [wan2-2](../wan2-2/_index.md)。 -- 依赖共享模块：[Diffusion 组件](../../components/diffusion/_index.md)。 - -## 结构与要点 - -- **硬依赖树外 `dreamid_omni` 包**（`fm_solvers`、`init_wan_vae_2_2`、 - `init_mmaudio_vae`、裁剪工具）——缺包直接 ImportError;树内代码不自足, - 改动/复现前先确认该 vendor 包版本（pin 上无 pin 记录）。 -- 双 VAE：`vae_model_video` + `vae_model_audio` 同入 `_vae_modules`; - `fusion.py` 的 `FusedBlock` 把 video block 与 audio block 配对做 layerwise - offload（`FusionModel` 带 checkpoint key remap）。 -- scheduler 在 forward 时可选 `FlowUniPCMultistepScheduler`/ - `FlowDPMSolverMultistepScheduler`/`FlowMatchEulerDiscreteScheduler`。 -- 类混入 `CFGParallelMixin, SupportImageInput, SupportAudioInput`。 - -## 什么时候查这里 - -- 审查 dreamid_omni 的双 VAE、fused offload 或外部包依赖变化。 +At the immediately preceding snapshot (`40dcfdbb`), the removed implementation +was a custom Wan2.2-based audio/video identity pipeline with an external +`dreamid_omni` dependency. This is historical context only: do not treat it as a +current registry target, supported checkpoint, runnable example, or review owner. +For the remaining Wan family, use [Wan2.2](../wan2-2/_index.md); for shared +runtime behavior, use [Diffusion](../../components/diffusion/_index.md).^[PR #6357] diff --git a/knowledge/repos/vllm-omni/models/magi-human/_index.md b/knowledge/repos/vllm-omni/models/magi-human/_index.md index 68aac912c..f8075b591 100644 --- a/knowledge/repos/vllm-omni/models/magi-human/_index.md +++ b/knowledge/repos/vllm-omni/models/magi-human/_index.md @@ -1,42 +1,21 @@ --- -title: "MagiHuman（音频驱动人像视频,单文件移植）" +title: "MagiHuman（已移除的 first-party 支持）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/magi_human/, vllm_omni/diffusion/registry.py] +sources: ["PR #6357"] --- -# MagiHuman +# MagiHuman — historical tombstone -以下事实在 `main @ 5d44868e` 复核（源码派生页,尚无本模型的运行经验沉淀）。 +PR #6357 removed first-party MagiHuman support at `main @ aeb348d8`: its +`MagiHumanPipeline` registry and pre/post-processing entries, the +`vllm_omni/diffusion/models/magi_human/` implementation, the model-extras +declarations, tests, and supported-model/feature documentation are gone. -## 名称与范围 - -- SandAI daVinci-MagiHuman 移植：音频驱动人像视频,输出同步视频 + 44.1 kHz - 音频（post 返回 `{video, audio, audio_sample_rate: 44100, fps: 25}`）。 -- 无别名、无变体有据,树内未 pin checkpoint。diffusion registry: - `MagiHumanPipeline` → - （`magi_human`, `pipeline_magi_human`, `MagiHumanPipeline`）,pre 为 - pass-through,post - `get_magi_human_post_process_func`。单 stage diffusion,引擎默认 stage 配置 - （[Config 组件](../../components/configuration/architecture.md)）。无 deploy YAML。 -- 源码仅 3 文件但全尾部最大：`pipeline_magi_human.py` **96 KB** + - `magi_human_dit.py` 62 KB（头注：移植自 daVinci-MagiHuman - `dit_module.py`,去掉了 Ulysses context-parallel）。 - -## 结构与要点 - -- **双 DiT**：`_dit_modules = ["dit", "sr_dit"]`（基座 + 超分）;VAE 双份—— - `DistributedAutoencoderKLWan` 视频 + 文件内 Oobleck 风格 `_AudioAutoencoder` - （`_vae_modules = ["vae", "audio_vae"]`,`pipeline_magi_human.py:1684`）。 -- 音频条件直接用 OpenAI `whisper`（`SAAudioFeatureExtractor`, - `whisper.load_audio`/`pad_or_trim`）;`_T5GemmaEncoder` 文本编码器;文件内 - 移植版 `FlowUniPCMultistepScheduler`（line 72）;内置超长音质负向 prompt。 -- 单文件 monolith + 自带 `MagiDataProxy` 数据代理——跨家族 refactor 评审时 - 把这份 96 KB 单文件移植列入检查清单。 - -## 什么时候查这里 - -- 审查 magi_human 的双 DiT/双 VAE、whisper 条件或 UniPC 移植改动。 -- 共享 RNG/graph 规则见 [Diffusion rules](../../components/diffusion/rules.md)。 +At the immediately preceding snapshot (`40dcfdbb`), it was an audio-driven +portrait-video pipeline with base and super-resolution DiTs plus audio/video +VAEs. This is historical context only: it is not an active pipeline, support +claim, model-extras contract, runnable example, or review owner. Shared runtime +questions belong to [Diffusion](../../components/diffusion/_index.md).^[PR #6357] From fe94797be36e57c47e16e018119e11a8545f2b2d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 20:00:11 +0800 Subject: [PATCH 310/465] knowledge(vllm-omni): sync 7ae15c74 (PR #6150) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../benchmark/guides/performance-evidence.md | 16 +++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../components/diffusion/rules-metrics.md | 39 +++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 11 ++---- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 17 +++++++- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/components/serving/rules.md | 21 ++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 14 files changed, 102 insertions(+), 23 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-metrics.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 081cfcfcb..2d5abba74 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 40dcfdbb4343fca917b34287cba620eb436a856d + previous_audited_sha: aeb348d8f8a555f0f9e91bee49f48778744a7bc6 audited_ref: main - audited_sha: aeb348d8f8a555f0f9e91bee49f48778744a7bc6 + audited_sha: 7ae15c74c8db75861478c989447639979e1fe617 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 35697198c..5601e617f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -aeb348d8f8a555f0f9e91bee49f48778744a7bc6`. The canonical machine baseline +7ae15c74c8db75861478c989447639979e1fe617`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/benchmark/guides/performance-evidence.md b/knowledge/repos/vllm-omni/benchmark/guides/performance-evidence.md index 8a2e526a3..b1a1dcdbd 100644 --- a/knowledge/repos/vllm-omni/benchmark/guides/performance-evidence.md +++ b/knowledge/repos/vllm-omni/benchmark/guides/performance-evidence.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-07-20 type: guide tags: [vllm-omni, benchmark] -sources: ["PR #5052"] +sources: ["PR #5052", "Issue #5811", "PR #6150"] --- # 性能与精度证据 @@ -66,6 +66,20 @@ sources: ["PR #5052"] - 功能 smoke 图如果质量差，不贴；用日志证明 grouping 即可。 5. 如果发现结果来源错了，不能用注释补救；必须撤掉旧表，重跑正确口径并替换。 +## 仪表盘 metric 的证据边界 + +Prometheus family 的实现、unit test 或 scrape 成功只证明测量路径的 wiring；不证明吞吐、 +latency 改善、显存收益、图像质量或模型质量。将新的 image/diffusion service metrics 写入 +性能或评估结论前，Evidence Matrix 还必须锁定 collector 是否由 `--log-stats` 启用、profile +开关、metric 的时间边界、缺失 versus 实测零、Prometheus query/window 和与 request/result +artifact 的对应关系。 + +尤其不要把 `stage_gen_time`、scheduler/request queue wait、service request time、forward +denoise time 或 `*_per_step` 当作同一个量：前者可包括 queue/async wait，纯 forward 又可能在 +profiler off 时 unavailable。PR #6150 留有一个未解决的 inline review：默认 profiler-off +路径的 benchmark-side `denoise_step_latency_ms` 可能显示为零。因此该字段在修复或明确标记 +unavailable 前不能用于比较、回归阈值或“无 denoise 耗时”的结论。^[Issue #5811] ^[PR #6150] + **2026-05-21 HunyuanImage3 DiT grouped batching 反例**： 我先把 DiT-only 英文 prompt 的性能表写成 PR 主性能结论： diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8eabd832b..3b0cbe4fc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ aeb348d8` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7ae15c74` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 285471f8a..065f59060 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ aeb348d8` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 7ae15c74` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -30,6 +30,7 @@ sources: [] |---|---| | 理解共享职责和数据流 | [architecture](architecture.md) | | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | +| image/diffusion timing、step normalization、missing-versus-zero 与 benchmark 解释边界 | [metrics evidence rules](rules-metrics.md) | | vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-metrics.md b/knowledge/repos/vllm-omni/components/diffusion/rules-metrics.md new file mode 100644 index 000000000..1cb26de97 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-metrics.md @@ -0,0 +1,39 @@ +--- +title: "Diffusion 指标证据规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["Issue #5811", "PR #4755", "PR #6150", vllm_omni/diffusion/outputs.py, vllm_omni/diffusion/sched/diffusion_scheduler.py, vllm_omni/metrics/prometheus.py] +confidence: high +--- + +# Diffusion 指标证据规则 + +本页承载 image/diffusion 请求指标的测量边界和解释合同。服务端的发射、聚合与 +replica 生命周期见 [Serving metrics rules](../serving/rules.md)；性能结论的证据门槛见 +[performance evidence](../../benchmark/guides/performance-evidence.md)。 + +## DIFF-5a — diffusion metrics 的 per-step 计算必须保留请求步数 + +- 触发：修改 diffusion output formatter、sampling params metadata 或 diffusion metrics accumulation，使请求级 DiT execution time 需要按 denoising step 归一化。 +- 强制：`format_diffusion_outputs()` 必须保留 `sampling_params.num_inference_steps` 并传入 stage stats；aggregator 保留该 scalar，仅在 execution time 与正步数同时存在时按 `exec_time / num_inference_steps` 观测 per-step metric。 +- 禁止：丢弃请求步数、用 stage 总生成时间替代 DiT execution time，或把跨多个结果的 scalar 累加后再计算 per-step 值。 +- 验收：覆盖完整 metadata、缺失/`None`/零步数、正步数、重复结果与零 execution time；分别断言 per-step 值精确、无效步数跳过且 scalar 不累加。^[PR #4755] + +## DIFF-5b — diffusion 指标必须区分 service time、forward time 与 unavailable + +- 触发：修改 diffusion scheduler admission、engine output `metrics`、profiler stage + durations，或将这些值汇总到 Prometheus/benchmark snapshots。 +- 强制：enqueue 到首次 `RUNNING` 的等待直接在 scheduler 测量；engine request time 从 + scheduler insertion 到收到 output，包含 scheduler wait、execution 和 async-output wait， + 但不含 preprocess/postprocess。只有 profiler `*.diffuse` duration 可用时才发射 + forward/denoise family；VAE、KV receive、peak memory 同样以实际 source 为前提。每个 + chunk 的 `*_ms` 先转秒再按 request 累加，per-step 仅在正 steps 下计算。 +- 禁止：用 `stage_gen - exec` 反推 queue time；把 service-time-per-step 称为纯 GPU + denoise；profiler 未启用时以 `0` 代替 unavailable，或让 benchmark snapshot 将缺失 + forward 值悄然显示为零。 +- 验收：覆盖 queue admission、async output、profile on/off、缺失 KV/VAE/memory、正/零 + steps 和多 output 累加；benchmark consumer 也必须明确区分 unavailable 与实测零。 + PR #6150 的最后一项仍是未解决 inline finding，不能据其测试或实现推断速度、质量或 + 模型效果。^[Issue #5811] ^[PR #6150] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index abc787855..9afbc9643 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] confidence: high --- @@ -34,6 +34,7 @@ confidence: high | Wan VAE spatial shard、gather/trim/reshard、empty tail、attention extent | `distributed-vae`：`DIFF-3b` | `distributed/autoencoders/wan_spatial_shard.py` → patched decoder attention/conv → gather final frame | | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4j` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | +| image/diffusion service time、forward time、missing-versus-zero、per-step metric | [`metrics-evidence`](rules-metrics.md)：`DIFF-5a`, `DIFF-5b` | output formatter/sampling metadata → stage stats aggregation → serving Prometheus emission | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -43,6 +44,7 @@ confidence: high | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | | `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j`（`4a`–`4i` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)） | +| [`metrics-evidence`](rules-metrics.md) | diffusion timing、step normalization、missing-versus-zero、benchmark interpretation | `DIFF-5a`, `DIFF-5b` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | ## 优化路径与 eager 的等价合同 @@ -178,13 +180,6 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl 证明质量差异来自目标量化变量。低 steps smoke 只能证明 runtime compatibility，不得声称 BF16 quality parity。 ^[PR #5136] ^[PR #6279] -### DIFF-5a — diffusion metrics 的 per-step 计算必须保留请求步数 - -- 触发：修改 diffusion output formatter、sampling params metadata 或 diffusion metrics accumulation，使请求级 DiT execution time 需要按 denoising step 归一化。 -- 强制：`format_diffusion_outputs()` 必须保留 `sampling_params.num_inference_steps` 并传入 stage stats；aggregator 保留该 scalar，仅在 execution time 与正步数同时存在时按 `exec_time / num_inference_steps` 观测 per-step metric。 -- 禁止：丢弃请求步数、用 stage 总生成时间替代 DiT execution time，或把跨多个结果的 scalar 累加后再计算 per-step 值。 -- 验收：覆盖完整 metadata、缺失/`None`/零步数、正步数、重复结果与零 execution time；分别断言 per-step 值精确、无效步数跳过且 scalar 不累加。^[PR #4755] - ### DIFF-4m — diffusion worker spawn 前必须限制继承的线程数 - 触发：修改 `MultiprocDiffusionExecutor._launch_workers`、worker spawn 流程或 diffusion 多进程的 OMP/Torch 线程环境。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 07c85d7e3..1acfbcc9d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ aeb348d8` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7ae15c74` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 065afaca7..b7c7770f3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ aeb348d8` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7ae15c74` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b69909632..dfe8b113c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ aeb348d8` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 7ae15c74` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 6c4cab9aa..b78dd2fdd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6149", "PR #6360", "PR #6406"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6149", "PR #6360", "PR #6406", "PR #6150"] --- # Scheduler 规则 @@ -26,6 +26,7 @@ PR 描述先命中下表，再打开对应规则组和首批源码；changed fil | side-stream D2H、pinned host tensor、源 buffer 复用 | SCHED-4a/4b | `worker/gpu_ar_model_runner.py::_copy_tensor_payload_to_cpu`、`_get_or_create_omni_payload_copy_stream`；`core/prefix_cache.py` 的 async copy 路径 | | sampled-token logprobs、spec decode trim、request-local output error | SCHED-5a | `core/sched/omni_ar_scheduler.py::_slice_sampled_logprobs`、`update_from_output` | | stateful async chunk、full-payload input、KV cleanup | SCHED-5b/5c | `core/sched/omni_generation_scheduler.py`、`omni_scheduling_coordinator.py`、`omni_ar_scheduler.py::_free_request` | +| KV extraction wait、connector acknowledgement、partial interval cleanup | SCHED-5h | `core/sched/omni_ar_scheduler.py::{_free_request,update_from_output}` → scheduler stats → orchestrator metrics | 若描述只写模型症状，先从模型 owner 找到 payload producer/consumer；只有实际断点落在调度、 prefix cache 或 copy lifetime 时才把 Scheduler 加为 owner。 @@ -231,6 +232,20 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 验收：分别覆盖 AR/generation scheduler 的成功、decoder 抛异常、无 decoder 和同批健康 request；断言成功 payload、`FinishReason.ERROR`、request-local 错误归属及无跨请求污染。 ^[PR #4795] +## SCHED-5h — KV extraction wait 只能观测完整区间 + +- 触发：AR scheduler 进入/离开 `waiting_for_transfer_free`，或传递 KV transfer + acknowledgement、connector type、abort/error cleanup metric。 +- 强制：在 `_free_request()` 进入 KV-transfer-free wait 时以 monotonic clock 记录 start； + 仅在 extraction acknowledgement 到达时 emit completed duration，并携带实际 connector + type。任何 finish、abort、error 或 request removal 都清除未完成 start，不能输出 partial + wait。 +- 禁止：以 request wall-clock、enqueue time 或 cleanup time 补造 KV wait；把没有 start + 的 acknowledgement 当零秒样本；因 telemetry 给 terminal request 保留 scheduler state。 +- 验收：覆盖 complete acknowledgement、无 start、abort/error/normal finish 清理和 iterator + request-id 输入；断言只有完整 wait 有一个样本，connector label 有界且没有 stale start。 + ^[PR #6150] + ## SCHED-5f — async-chunk 等待必须有可配置的截止时间 - 触发：async-chunk 请求进入 `WAITING_FOR_CHUNK`，或部署配置读取 `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S`。 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 56b25cb10..a3a9972ac 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ aeb348d8` 验证存在 +- 源码校验：以上路径均已在 `main @ 7ae15c74` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index b61fe4d5e..16f470d1e 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288"] confidence: high --- @@ -33,14 +33,14 @@ confidence: high | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | | video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | -| Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle | `metrics-lifecycle`：`SERV-2a`, `SERV-2b` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | +| Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle、image/diffusion metric emission | `metrics-lifecycle`：`SERV-2a`, `SERV-2b`, `SERV-2e` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| | `core` | 每次 serving 审查 | `SERV-4c`，见 [请求输入合同](rules-request-input.md) | | `streaming-format` | SSE、audio streaming、format/default/capability | `SERV-1a`, `SERV-1b` | | `media-ingress` | video reference、decoder registry/backend、mixed capability、bounded upload/conversion | `SERV-1c`–`1e` | -| `metrics-lifecycle` | metrics、gauge、replica、collector | `SERV-2a`, `SERV-2b` | +| `metrics-lifecycle` | metrics、gauge、replica、collector、image/diffusion measurement boundary | `SERV-2a`, `SERV-2b`, `SERV-2e` | | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | @@ -181,6 +181,21 @@ confidence: high - 禁止：在轮询重试、重复回调或消息标记后再次累计同一 result；把同一消息误当作新的 denoising step，或绕过现有 consumed 生命周期清理。 - 验收：同一 result 重复处理只产生一次累计，两个 distinct result 各自产生一次；覆盖异常、重试与请求清理，确认 consumed 状态不会导致重复累计或遗留。^[PR #4755] +## SERV-2e — 指标发射必须保持测量边界、缺失语义与请求唯一性 + +- 触发：新增 image/diffusion Prometheus family，或修改 `OmniBase` terminal result、 + `OrchestratorAggregator`、scheduler snapshot、失败清理或 stage/replica membership。 +- 强制：只在同一 finished result 的首次消费中发射 stage workload；request-finalize + family 由 e2e guard 发射一次。保留已测得的零 queue duration，但没有来源的 optional + profiler/KV/memory measurement 不发射。failure counter 在 abort、disconnect、stage error + 和 cleanup 之间去重，并将 reason 归一到有界 taxonomy。waiting gauge 只求 live replicas + 的最新快照，且 abort/error、death 和正常 unregister 都必须刷新或移除旧快照。 +- 禁止：把缺失值补成零、重放 terminal message 后重复加 Counter/Histogram、把已移除 + replica 的历史 waiting 值留在总和，或把 `stage_gen_time` 当纯 DiT forward 时间。 +- 验收：覆盖 finished-result replay、present-zero 与 missing、错误/abort cleanup、正常及 + dead replica unregister、一次请求的多个 failure path，以及 image-only workload guard。 + profiler-off 与 profiler-on 均须断言各自可观测合同。^[Issue #5811] ^[PR #6150] + ## SERV-2d — 图像生成与编辑响应必须透传 diffusion 指标 - 触发：修改 `/v1/images/generations` 或 `/v1/images/edits` 的 diffusion generation result 解包、指标字段或 OpenAI 响应构造。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 517c221e1..23c0aee97 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ aeb348d8`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7ae15c74`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 5bebaf6e00d1f9b692cff2cec093c7bd348ef80a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 20:13:24 +0800 Subject: [PATCH 311/465] knowledge(vllm-omni): sync 5de6f7c3 (PR #6619) --- adapters/vllm_omni/release_baseline.yaml | 8 +++--- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/rules.md | 12 +++------ .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 21 ++++++++++------ .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 7 ++++-- .../vllm-omni/models/minicpm-o-4-5/_index.md | 3 ++- .../models/minicpm-o-4-5/rules-duplex.md | 25 +++++++++++++++++++ .../vllm-omni/models/minicpm-o-4-5/rules.md | 11 +++----- 14 files changed, 62 insertions(+), 39 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2d5abba74..f9e332738 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: aeb348d8f8a555f0f9e91bee49f48778744a7bc6 + previous_audited_sha: 7ae15c74c8db75861478c989447639979e1fe617 audited_ref: main - audited_sha: 7ae15c74c8db75861478c989447639979e1fe617 + audited_sha: 5de6f7c37afdd189e010a46ba6b96fcc14f6388f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -19,8 +19,8 @@ inventories: count: 55 sha256: 9d5033c9a7ffd7ff636e32db0f9be2a549b7c16b034e13817e280f19a9e16769 deploy_yamls: - count: 84 - sha256: 002a24e33c1369f7e1b0a6824d0d2805c6ffb4791c652d6d67fd161a075017bd + count: 83 + sha256: a229dcbee310f9ffe511ef103e09f278e626761a61abe658f866aa0a52564b85 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5601e617f..55a7cfb3c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -7ae15c74c8db75861478c989447639979e1fe617`. The canonical machine baseline +5de6f7c37afdd189e010a46ba6b96fcc14f6388f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3b0cbe4fc..ca5f2222b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7ae15c74` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5de6f7c3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index d5d126c6e..bd2c3db97 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820", "PR #6619"] --- # vLLM-Omni 配置开发门禁 @@ -22,6 +22,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR |---|---|---| | strict schema、unknown field、alias、flat→nested、structured/legacy/direct parity、typed projection | `strict-normalization`：`VOMNI-CFG-1a`–`1h` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,strip_parent_engine_args}` → `vllm_omni/config/omni_config.py::{_build_diffusion_config_projection,VllmOmniConfig.from_pipeline_config}` → `vllm_omni/engine/stage_init_utils.py::{build_engine_args_dict,build_engine_args_dict_from_omni_stage_config}` | | deploy YAML、`base_config`、pipeline/stage overlay、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a` | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::{StageConfigFactory.create_from_model,StageConfigFactory._merge_cli_overrides}` | +| MiniCPM-o shipping profile、duplex/chat 共服、Talker sampling floor | model-specific [`MCPMO-4e`](../../models/minicpm-o-4-5/rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → `merge_pipeline_deploy`；duplex adapter runtime sampling override | | composable strategy、axis、routing、load balancing、`strategy-config` | `composable-strategy`：`CONF-4a` | `vllm_omni/config/composable_parallel/strategy_loader.py::{parse_strategy_specs,load_strategy_specs}` → `translator.py::translate_strategy_stack` → `apply.py::apply_strategy_specs` → `config_factory.py::{StageConfigFactory._apply_strategy_specs,StageConfigFactory._reconcile_strategy_with_cli}` | | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | | multi-stage HF sub-config、stage quantization view、`hf_config_name` | `stage-model-config`：`VOMNI-CFG-1i` | pipeline stage declaration → `OmniModelConfig::{draw_hf_text_config,get_model_arch_config}` → vLLM quantization selection | @@ -33,7 +34,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `core` | 每次配置审查 | `VOMNI-CFG-1b`, `VOMNI-CFG-1c` | | `strict-normalization` | schema、alias、unknown field、structured/legacy/direct 路径 | `VOMNI-CFG-1a`, `VOMNI-CFG-1b`, `VOMNI-CFG-1c`, `VOMNI-CFG-1d`, `VOMNI-CFG-1e`, `VOMNI-CFG-1f`, `VOMNI-CFG-1g`, `VOMNI-CFG-1h` | | `deploy-memory` | 显存预算、KV pin、多 stage 共卡 | `CONF-1a`, `CONF-2a` | -| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5g` 见 [topology 与部署 profile](rules-topology-profiles.md) | +| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5g` 见 [topology 与部署 profile](rules-topology-profiles.md)；MiniCPM-o profile 见 `MCPMO-4e` | | `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a`, `CONF-4c`–`4d`，见 [并行拓扑合同](rules-parallel-topology.md) | | `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | | `stage-config-propagation` | pipeline sampling constraints、StageConfig serialization/runtime config | `VOMNI-CFG-1o` | @@ -197,13 +198,6 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：只注册 pipeline 或放置 YAML 就依赖 `model_type` 推断；用 `hf_overrides.model_type` 替代 stage architecture 路由；要求离线入口额外传参来弥补默认配置未接通。 - 验收：真实无显式 deploy 配置的初始化能读回最终 pipeline 与 architecture，backbone `get_text_config()` 仍解析为原模型类型，并覆盖 registry、默认 YAML 与最终 stage config 的一致性。 ^[PR #5666] -### CONF-3e — MiniCPM-o 4.5 Stage 1 默认采样参数必须与模型语义一致 - -- 触发：修改 `minicpmo_4_5.yaml`、`minicpmo_4_5_2gpu.yaml`、`minicpmo_4_5_3gpu.yaml` 或 `minicpmo_4_5_8x4090.yaml` 的 Stage 1 Talker 默认采样参数。 -- 强制：四份 profile 的 Stage 1 默认值保持一致：`temperature=0.8`、`top_p=0.85`、`top_k=25`、`repetition_penalty=1.05`、`min_tokens=50`、`max_tokens=4096`；duplex overlay 可按 `generate_chunk` 合同将 `min_tokens` 设为 `0`，而 Talker 离线逻辑仍独立限制为 2048 或剩余上下文。 -- 禁止：只更新一个 profile；把旧的 whole-stream penalty 值 `1.02` 沿用到 16-frame frequency penalty；将 Sampler ceiling `4096` 解释为离线 Talker 的实际生成预算；把 `min_p`、`win_size` 或 `tau_r` 当作已由上游 `gen_logits()` 消费的必需字段。 -- 验收：展开四份 MiniCPM-o 4.5 deploy 配置，逐一断言 Stage 1 的采样字段与上游默认一致，并单独验证 duplex overlay 的 `min_tokens=0` 不改变其余继承值；真实 Talker 测试还需确认最终配置到达 Sampler 与模型预算逻辑。^[PR #6458] - ### CONF-6a — MOSS-TTS-Local 部署必须采用官方采样参数 - 触发：修改 MOSS-TTS-Local 的 deploy YAML 或其默认采样参数时。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 065f59060..e42e0580d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7ae15c74` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5de6f7c3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1acfbcc9d..17c25e59a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7ae15c74` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5de6f7c3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b7c7770f3..a97d6173a 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7ae15c74` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5de6f7c3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index dfe8b113c..e828fa59c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7ae15c74` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5de6f7c3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index b78dd2fdd..346ab5718 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6149", "PR #6360", "PR #6406", "PR #6150"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619"] --- # Scheduler 规则 @@ -27,6 +27,7 @@ PR 描述先命中下表，再打开对应规则组和首批源码；changed fil | sampled-token logprobs、spec decode trim、request-local output error | SCHED-5a | `core/sched/omni_ar_scheduler.py::_slice_sampled_logprobs`、`update_from_output` | | stateful async chunk、full-payload input、KV cleanup | SCHED-5b/5c | `core/sched/omni_generation_scheduler.py`、`omni_scheduling_coordinator.py`、`omni_ar_scheduler.py::_free_request` | | KV extraction wait、connector acknowledgement、partial interval cleanup | SCHED-5h | `core/sched/omni_ar_scheduler.py::{_free_request,update_from_output}` → scheduler stats → orchestrator metrics | +| async stop、streaming update、stale token fence、next duplex unit 被吞 | SCHED-6a | `core/sched/omni_ar_scheduler.py::{_handle_stopped_request,update_from_output}` → `test_omni_ar_scheduler_streaming.py` | 若描述只写模型症状，先从模型 owner 找到 payload producer/consumer；只有实际断点落在调度、 prefix cache 或 copy lifetime 时才把 Scheduler 加为 owner。 @@ -263,13 +264,17 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 ## SCHED-6a — async discard 的计数单位必须与 stale drain 一致 - 触发：上游 scheduler 改动异步占位、stale output 或 streaming segment replacement。 -- 强制：每个 discard site 用 `num_in_flight_tokens`（scheduled-token 单位）累加 - `num_stale_output_tokens`；每个迟到 frame 先按本次 `num_tokens_scheduled` 结算 in-flight 与 stale - 两个同单位 counter，再在 append/emit 前丢弃。placeholder rollback 与 stale drain 分开记账。 -- 禁止：用 `num_output_placeholders` 初始化 stale counter；清零 placeholder 后仍让迟到输出进入 - placeholder/computed-token 的正常结果更新；把 scheduler 修复宣称为音频 WER 修复。 -- 验收：AR 与 generation 路径覆盖多 token frame、prefill 无 placeholder、连续旧/新 segment， - 断言 stale counter 恰好归零且首个新 segment 输出不被吞；真实质量指标必须另行验证。^[PR #5976] +- 强制：每个 discard site 用 `num_in_flight_tokens`（scheduled-token 单位）建立 + `num_stale_output_tokens`；同一 in-flight decode 若已由 queued streaming update fence，stop/replacement + site 必须赋值而非累加，避免残留吞掉下一 duplex unit。每个迟到 frame 先按本次 + `num_tokens_scheduled` 结算 in-flight 与 stale 两个同单位 counter，再在 append/emit 前丢弃。 + placeholder rollback 与 stale drain 分开记账。 +- 禁止：用 `num_output_placeholders` 初始化 stale counter；对已由同次 streaming update fenced 的 + in-flight token 再累加；清零 placeholder 后仍让迟到输出进入 placeholder/computed-token 的正常 + 结果更新；把 scheduler 修复宣称为音频 WER 修复。 +- 验收：AR 与 generation 路径覆盖多 token frame、prefill 无 placeholder、连续旧/新 segment 和 + queued update 后的 async stop，断言 stale counter 恰好归零且首个新 segment 输出不被吞；真实质量 + 指标必须另行验证。^[PR #5976] ^[PR #6619] ## 相关 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a3a9972ac..534e73dbf 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7ae15c74` 验证存在 +- 源码校验：以上路径均已在 `main @ 5de6f7c3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 23c0aee97..7ef2d1a2e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7ae15c74`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5de6f7c3`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -38,7 +38,7 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | | Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 40 个模型族目录 | | Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 55 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 84 份 | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 83 份 | 对比上一审计快照（`39c16d75`）：AR 架构 88→81，diffusion pipeline 65→64， OMNI_PIPELINES 57→55，deploy 86→84。PR #6353 删除 AudioX 的 diffusion registry、 @@ -53,6 +53,9 @@ first-party 支持，下面的 registry/owner 清单不将其列为 active。两 不得作为可运行支持或能力证据。RFC #5996 最初讨论的是 CI delist；本 PR 的合并代码和 PR body 明确扩大为完整 first-party 删除。^[PR #6357] ^[Issue #5996] +PR #6619 删除独立的 `minicpmo_4_5_duplex.yaml`，将 duplex 配置并入全部 shipping MiniCPM-o +profile，因此 deploy inventory 84→83；这是 profile 合并，不是 MiniCPM-o 能力删除。^[PR #6619] + PR #4820 将 `WanDMDPipeline` 注册为 Wan2.2 的同实现入口，因此 diffusion pipeline 计数回升至 65；它不是新的模型族或新的 `OMNI_PIPELINES` key。^[PR #4820] diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 39b887725..e31ba5fb3 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -50,7 +50,8 @@ Online serving CI 的 core suite 固定 async chunk，expansion 分开覆盖 syn fixture 不再覆盖 eager/KV cap 并固定 active-speech turn window；验证证据边界见 MCPMO-5b。 描述直达源码与模型专有门禁见 [rules](rules.md#direct-代码快速入口)；并发 duplex 的 ragged -Code2Wav state、batch cap 与 NPU residual limits 见 [Code2Wav 并发批处理规则](rules-code2wav-batching.md)。新模型语义验证见 +Talker chunk、shipping profile 与 chat 隔离见 [native duplex 规则](rules-duplex.md)；Code2Wav +state、batch cap 与 NPU residual limits 见 [Code2Wav 并发批处理规则](rules-code2wav-batching.md)。新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md new file mode 100644 index 000000000..41f2cfc5c --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -0,0 +1,25 @@ +--- +title: "MiniCPM-o 4.5 native duplex 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, model-executor] +sources: ["PR #6346", "PR #6458", "PR #6619", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, tests/config/test_config_factory.py, tests/entrypoints/openai_api/test_duplex_handler.py] +confidence: high +--- + +# MiniCPM-o 4.5 native duplex 规则 + +## MCPMO-4c — native duplex Talker 必须按 generate_chunk 预算终止 + +- 触发：修改 MiniCPM-o native duplex Talker 的 `generate_chunk`、codec EOS 或请求级音频状态。 +- 强制：每个 native duplex 请求按一次 chunk 的 26 个采样管理状态；adapter 只在 duplex runtime config 将 Talker `min_tokens` 设为 0，保留 chat YAML 的 whole-utterance floor；turn 边界 `min_tokens=0`，中间 chunk `min_tokens=max_tokens=26`，已转发 25 帧后强制下一个采样为 EOS，仅把非 EOS code 交给 Code2Wav。 +- 禁止：只依赖 YAML `max_tokens` 维持 turn 边界；改变 chat 的 whole-utterance Talker floor；在中间 chunk 永久屏蔽 EOS；把终止采样再次作为普通 codec frame 输出。 +- 验收：覆盖 prefill 边界元数据、step 24/25 的 mask/force EOS、runtime Stage 1 `min_tokens=0`、chat sampling control、simplex 不误触发以及多请求 delta/finished 对齐。^[PR #6346] ^[PR #6619] + +## MCPMO-4e — shipping profile 必须共服 chat 与 native duplex + +- 触发：修改任一 `minicpmo_4_5*.yaml`、duplex session placement 或 Stage 1 Talker 默认/运行时采样参数。 +- 强制：所有 shipping profile 设置 `session_mode=duplex`、`active_stream_window=1`，且 `duplex_session.max_sessions` 与 Stage 0/1 的 `max_num_seqs` 容量一致；replica overlay 从 base 继承这些字段。YAML 保留 chat 的 Stage 1 `min_tokens=50` 与既有 codec 参数，native duplex adapter 仅在该 session 的 runtime config 覆盖 `min_tokens=0`。AR stage 保留默认 async scheduler。 +- 禁止：恢复独立 `minicpmo_4_5_duplex.yaml`；在 replica overlay 重复并漂移 session 配置；为 duplex 全局关闭 async scheduling；或把 runtime-only floor 写回 YAML 而改变 chat TTS。 +- 验收：展开四份 base profile 及三个 replica overlay，断言全部 duplex-enabled、窗口与 session capacity 正确；同一 server 分别验证 `/v1/realtime?duplex=1` 和 chat，前者 Stage 1 看到 `min_tokens=0`、后者仍看到 `50`。删除旧 overlay 是部署文件名迁移，外部显式引用必须同步更新。^[PR #6458] ^[PR #6619] diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 3d4559cd9..f8a3c3442 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6619"] confidence: high --- @@ -27,6 +27,8 @@ confidence: high | Talker codec sampling、repetition penalty、request RNG/compaction | MCPMO-3c | `minicpmo_4_5_omni_tts.py::{make_omni_output,_sample_audio_codes,_apply_batched_repetition_penalty}` → `test_talker_batching.py` | | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | | instructions/persona/voice/mode update、prefill slots、context lock | MCPMO-4b | `experimental/fullduplex/minicpmo45/session.py` → runtime adapter/session runner | +| duplex Talker chunk、runtime `min_tokens`、chat sampling isolation | [`MCPMO-4c`](rules-duplex.md#mcpmo-4c-native-duplex-talker-必须按-generate_chunk-预算终止) | `experimental/fullduplex/minicpmo45/adapter.py` → stage 1 sampling → Talker `generate_chunk` | +| shipping YAML、duplex/chat 共服、session capacity、async scheduler | [`MCPMO-4e`](rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → config merge → realtime/chat entrypoints | | Daily-Omni/Seed-TTS accuracy、simplex/duplex perf、TTFT/TTFP/RTF | MCPMO-5a | `tests/e2e/accuracy/minicpmo_4_5/`、`tests/dfx/perf/tests/test_minicpmo_4_5*.json` → `run_benchmark.py` | | online serving CI、async/sync chunk matrix、duplex fixture、local ref audio | MCPMO-5b | `tests/e2e/online_serving/test_minicpmo_4_5*.py` → `helpers/minicpmo_4_5_duplex.py` | @@ -249,13 +251,6 @@ confidence: high - 验收：成功更新同时改变派生状态；资源不足时无部分 mutation；buffer/defer/reject 均不锁， append 成功才锁，mode flip 明确拒绝。 ^[PR #6318] -## MCPMO-4c — native duplex Talker 必须按 generate_chunk 预算终止 - -- 触发：修改 MiniCPM-o native duplex Talker 的 `generate_chunk`、codec EOS 或请求级音频状态。 -- 强制：每个 native duplex 请求按一次 chunk 的 26 个采样管理状态；turn 边界 `min_tokens=0`，中间 chunk `min_tokens=max_tokens=26`，已转发 25 帧后强制下一个采样为 EOS，仅把非 EOS code 交给 Code2Wav。 -- 禁止：只依赖 YAML `max_tokens` 维持 turn 边界；在中间 chunk 永久屏蔽 EOS；把终止采样再次作为普通 codec frame 输出。 -- 验收：覆盖 prefill 边界元数据、step 24/25 的 mask/force EOS、simplex 不误触发以及多请求 delta/finished 对齐。 ^[PR #6346] - ## MCPMO-4d — async chunk 快照替换必须清理旧音频 - 触发：MiniCPM-o 4.5 native duplex 遇到重复 `turn_end` 或新的 Code2Wav async segment。 From 1132ce88acba62efd2d708909f7a834cd68854ad Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 20:19:01 +0800 Subject: [PATCH 312/465] knowledge(vllm-omni): sync f90e992d (PR #5673) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 1 + .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../vllm-omni/models/cosyvoice3/_index.md | 5 +-- .../models/cosyvoice3/architecture.md | 12 ++++++- .../vllm-omni/models/cosyvoice3/rules.md | 32 +++++++++++++++++++ 13 files changed, 57 insertions(+), 13 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/cosyvoice3/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f9e332738..0a2f95374 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7ae15c74c8db75861478c989447639979e1fe617 + previous_audited_sha: 5de6f7c37afdd189e010a46ba6b96fcc14f6388f audited_ref: main - audited_sha: 5de6f7c37afdd189e010a46ba6b96fcc14f6388f + audited_sha: f90e992dc1dd45ed5cc3c536ee39567ef3f53470 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 55a7cfb3c..bee7f282d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5de6f7c37afdd189e010a46ba6b96fcc14f6388f`. The canonical machine baseline +f90e992dc1dd45ed5cc3c536ee39567ef3f53470`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ca5f2222b..bd409d96c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5de6f7c3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f90e992d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e42e0580d..1ed5a2faa 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5de6f7c3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f90e992d` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 17c25e59a..6d31422a4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5de6f7c3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f90e992d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a97d6173a..c9954b8d6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5de6f7c3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f90e992d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index fa6c9b574..1aa6f500d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -27,6 +27,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | Talker-MTP FULL graph、平台 capture 能力、显式 opt-out | `mtp-graph`：`EXEC-4c` | `platforms/interface.py::supports_talker_mtp_graph_capture` → platform override → model `talker_mtp_graph_safe` → `OmniGPUModelRunner._init_talker_mtp` | | `model_extras`、shared T2I/I2V example、canonical prompt envelope | `image-task-envelope`：`EXEC-6a` | `examples/offline_inference/{text_to_image/text_to_image.py,image_to_video/image_to_video.py}` → `model_extras/registry.py::{build_text_to_image_prompt,build_image_to_video_prompt}` → model pipeline validation | +| CosyVoice3 non-module TensorRT CFM、stream/event ownership、raw-pointer buffer lifetime 或 context-pool reuse | [CosyVoice3 TensorRT handoff](../../models/cosyvoice3/rules.md)：`COSYVOICE3-1a` | `code2wav_core/cfm.py::ConditionalCFM.forward_estimator` → `flow_estimator_trt.py::TrtContextWrapper` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e828fa59c..be09a492c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5de6f7c3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f90e992d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 534e73dbf..b9b29c8b2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5de6f7c3` 验证存在 +- 源码校验：以上路径均已在 `main @ f90e992d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 7ef2d1a2e..4b2d0be67 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5de6f7c3`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f90e992d`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md b/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md index e9a40c560..44a5e60d7 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md @@ -4,12 +4,12 @@ created: 2026-07-21 updated: 2026-09-04 type: index tags: [vllm-omni, models] -sources: ["PR #5869", "PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoice3.yaml, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py] +sources: ["PR #5673", "PR #5869", "PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoice3.yaml, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py] --- # CosyVoice3 -以下事实在 `main @ 39c16d75` 复核。 +以下事实在 `main @ f90e992d` 复核。 ## 名称与范围 @@ -51,6 +51,7 @@ sources: ["PR #5869", "PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_ | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | RAS 停止机制、双交接注册、TRT 门 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | +| CosyVoice3 TensorRT CFM 的 stream handoff、allocator lifetime 或 context pool | [rules](rules.md) | `COSYVOICE3-1a` 的顺序、所有权与验收 | ## 配置与 checkpoint 差异 diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md b/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md index a7a386cc3..583de87a3 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-21 updated: 2026-09-04 type: architecture tags: [vllm-omni, models] -sources: ["PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/cosyvoice3/pipeline.py, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py, vllm_omni/deploy/cosyvoice3.yaml] +sources: ["PR #5673", "PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py, vllm_omni/model_executor/models/cosyvoice3/pipeline.py, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py, vllm_omni/deploy/cosyvoice3.yaml] --- # CosyVoice3 架构 @@ -23,6 +23,16 @@ sources: ["PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/mod - **TRT 加速（本清单独有）**：campplus 说话人嵌入与 CFM DiT 估计器都可走 TensorRT（启动时 ONNX→plan,按设备缓存 plan;campplus 有 CPU-ONNX 兜底）, 统一由 `COSYVOICE3_TRT` 门控,默认开。 +- **CFM TensorRT handoff**：每个 execution context 与一个原始 + `torch.cuda.Stream` 成对入池；caller stream 先由该 stream 的 `wait_stream` + 建立 producer→TensorRT 依赖，TRT 在该 stream 的 raw `cuda_stream` handle 上 + enqueue。返回前 caller 对 estimator stream 再 `wait_stream`，所以后续 CFG + buffer 原地写入和 output dtype conversion 在 consumer stream 上发生在 TRT 后。 + 所有供 raw pointer binding 的输入/输出都 record estimator stream；输出还 record + caller stream，分别覆盖 enqueue 前后 allocator reuse。context 可在 enqueue 后回池， + 因为它始终与专属 stream 成对且 TensorRT 已 snapshot 地址；这不是 host synchronize。 + `torch.nn.Module` estimator 路径和 `COSYVOICE3_TRT` 的既有关闭/CPU fallback + 不在这个非-module TensorRT 分支中改变。具体改动约束见 [rules](rules.md)。^[PR #5673] - 共享：diffusion Attention 层、SharedMemoryConnector、 [Config 组件](../../components/configuration/architecture.md)。 diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md b/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md new file mode 100644 index 000000000..f34b4c6d7 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md @@ -0,0 +1,32 @@ +--- +title: "CosyVoice3 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models] +sources: ["PR #5673", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py, vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py, tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py, benchmarks/tts/benchmark_cosyvoice3_trt_streams.py] +confidence: high +--- + +# CosyVoice3 规则 + +## COSYVOICE3-1a — TensorRT CFM 的跨 stream handoff 必须同时守住顺序和 allocator lifetime + +- 触发：修改 CosyVoice3 `ConditionalCFM.forward_estimator` 的非-`torch.nn.Module` + TensorRT 分支、`TrtContextWrapper` 的 context/stream pool，或 raw device-pointer + 输入/输出、dtype conversion 和 CUDA stream handoff。 +- 强制：pool 中保存每个 execution context 配对的原始 `torch.cuda.Stream`；在 estimator + stream 上先 `wait_stream(caller_stream)`，在其 raw `cuda_stream` handle 上 enqueue TRT， + 然后让 caller `wait_stream(estimator_stream)` 再消费输出。对每个以 raw pointer 绑定的 + CUDA input/output record estimator stream，且在返回/转换前对 output record caller stream。 + context 只有在地址已 enqueue 到其专属 stream 后才能归还同一个 pair。 +- 禁止：以 host `synchronize()` 替代任一依赖；把 `torch.cuda.stream(...)` 的 context + manager 当作 pool stream；只保护输出或只保护输入，使 caching allocator 能在 TRT 完成前 + 重用 `.to(...).contiguous()` buffer；在未建立 consumer wait 前复用 CFG 的原地写入 buffer。 + 不得把这个优化扩展为改变 `torch.nn.Module` estimator 路径，或静默改变 + `COSYVOICE3_TRT` 关闭和非-CUDA fallback。 +- 验收：CPU fake-stream 回归断言零 `synchronize`、两个方向各一次 `wait_stream`、TRT 接到 + estimator raw handle，且 context/stream pair 原样回池；真实 CUDA/TensorRT 覆盖 raw-pointer + input/output 的 allocator-lifetime 与数值 parity。性能报告要分开写 submission、engine 和 + E2E 指标：受控 handoff microbenchmark 只能证明消除了 host blocking，有限的 E2E 样本或 + engine 小幅变化不能声称模型吞吐/延迟提升。^[PR #5673] From eb718fb52935a51d00dc3e0b601eb17be601c406 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 20:25:42 +0800 Subject: [PATCH 313/465] knowledge(vllm-omni): sync 7ba5e8a1 (PR #6624) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 3 ++- .../rules-model-reference-routing.md | 21 +++++++++++++++++++ .../components/configuration/rules.md | 11 +++------- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../serving/rules-upstream-compat.md | 7 +++---- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 12 files changed, 38 insertions(+), 22 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/configuration/rules-model-reference-routing.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0a2f95374..a621c1707 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5de6f7c37afdd189e010a46ba6b96fcc14f6388f + previous_audited_sha: f90e992dc1dd45ed5cc3c536ee39567ef3f53470 audited_ref: main - audited_sha: f90e992dc1dd45ed5cc3c536ee39567ef3f53470 + audited_sha: 7ba5e8a146b41f7467caefa1795ecf6842b0077c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index bee7f282d..5d6f14f23 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f90e992dc1dd45ed5cc3c536ee39567ef3f53470`. The canonical machine baseline +7ba5e8a146b41f7467caefa1795ecf6842b0077c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bd409d96c..5cf4d1dad 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f90e992d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7ba5e8a1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 @@ -36,6 +36,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] |---|---| | 理解配置从 deploy、CLI、默认 factory 到 structured/legacy config 的稳定边界 | [配置构造架构](architecture.md) | | 根据 PR 描述直达 strict schema、pipeline sampling constraints、deploy/topology、composable strategy 或显存配置的规则组与第一批源码 | [配置开发门禁与代码地图](rules.md) | +| 对象存储 URI、HF cache snapshot 或空 `config.json` 的模型名称回退与 pipeline 路由 | [模型引用路由规则](rules-model-reference-routing.md) | | stage full-payload transport capability、topology-owned projection 或 deploy/CLI override rejection | [stage transport capability](rules-stage-transport.md) | | 审计配置来源和多层加工 | [config audit](config-audit-plain-language.md) | | 设计配置归一化与 parity 验证 | [config normalization parity](config-normalization-parity.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-model-reference-routing.md b/knowledge/repos/vllm-omni/components/configuration/rules-model-reference-routing.md new file mode 100644 index 000000000..e20eafc4c --- /dev/null +++ b/knowledge/repos/vllm-omni/components/configuration/rules-model-reference-routing.md @@ -0,0 +1,21 @@ +--- +title: "模型引用路由规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, config] +sources: [vllm_omni/config/config_factory.py, vllm_omni/config/pipeline_registry.py, tests/config/test_config_factory.py, "PR #5036", "PR #6624"] +--- + +# 模型引用路由规则 + +## Direct 代码快速入口 + +- **CONF-7a — 模型引用解析。** 从 `config_factory.py::{_materialize_object_storage_configs,_name_match_candidate,StageConfigFactory._try_infer_model_type}` 直达 `pipeline_registry.py::OMNI_PIPELINES`；cache-shape 回归从 `tests/config/test_config_factory.py::TestNameMatchCandidateSnapshotPaths` 开始。 + +### CONF-7a — 模型引用解析必须物化对象存储配置，并只从受控名称组件匹配 + +- 触发：父进程在 stage-specific `ModelConfig` 初始化前，需要从 `s3://`、`gs://` 或上游 `is_runai_obj_uri` 支持的对象存储 URI 推断模型类型、读取 HF 配置或解析 `model_index.json`。 +- 强制：通过 `is_runai_obj_uri` 将对象存储模型的 `*.model`、`*.py`、`*.json` 轻量文件按 URI 缓存一次到确定性的 `model_streamer/` 目录；`get_config`、`config.json` 和 `model_index.json` 的读取统一使用该本地目录，同时让各 stage 继续接收原始 URI 以保留 `model_weights` 的流式加载语义。最后的 name-based fallback 通常只比较末个路径组件；唯一例外是有效 HF hub cache snapshot 的尾形 `models----/snapshots/` 或 legacy `models--/snapshots/`，此时只恢复 repo 的 ``，不能让组织名参与匹配。 +- 禁止：把原始对象存储 URI 交给 Hugging Face 仓库解析；把临时 `model_streamer/` 路径传播成 stage 的模型身份；用完整 URI、bucket 或组织名参与模型类型名称匹配；把普通 `.../snapshots/` 或 malformed `models--...` 误认成 HF cache（它们必须继续取末个组件）。 +- 验收：用 mock 覆盖对象存储 URI 的单次物化、配置驱动的 pipeline 选择、`model_index.json` 回退和欺骗性 bucket 名；参数化覆盖 namespaced/legacy HF cache snapshot、trailing slash、普通路径与 malformed cache segment，断言只从允许的 repo-name 段恢复名称。空 `config.json` 的 snapshot 必须仍能由 `try_infer_model_type` 选中注册的 CosyVoice3 pipeline。PR #6624 的真实 H20 E2E 和 config-suite 结果由作者报告，未在本次知识审计环境复跑；其后续 review 还指出两个 consumer 注释仍称“basename-only”，这是文案漂移而非该提交已改动的行为。^[PR #5036] ^[PR #6624] diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index bd2c3db97..110156632 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #5036", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820", "PR #6619"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820", "PR #6619"] --- # vLLM-Omni 配置开发门禁 @@ -28,6 +28,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | multi-stage HF sub-config、stage quantization view、`hf_config_name` | `stage-model-config`：`VOMNI-CFG-1i` | pipeline stage declaration → `OmniModelConfig::{draw_hf_text_config,get_model_arch_config}` → vLLM quantization selection | | pipeline `sampling_constraints`、`StageConfig.to_omegaconf()`、runtime stage config | `stage-config-propagation`：`VOMNI-CFG-1o` | `merge_pipeline_deploy` → `StageConfig.to_omegaconf()` → `engine.stage_configs` | | stage transport、`requires_full_payload_input`、topology projection 或 override rejection | `stage-transport`：`VOMNI-CFG-1p` | `stage_config.py::{StagePipelineConfig,_build_engine_args}` → `omni_config.py::_build_model_config` → `engine/arg_utils.py::OmniEngineArgs` | +| HF cache snapshot path、空 `config.json`、name-based pipeline fallback | [`model-reference-routing`](rules-model-reference-routing.md#conf-7a-模型引用解析必须物化对象存储配置并只从受控名称组件匹配)：`CONF-7a` | `config_factory.py::{_name_match_candidate,StageConfigFactory._try_infer_model_type}` → `pipeline_registry.py::OMNI_PIPELINES` → `tests/config/test_config_factory.py::TestNameMatchCandidateSnapshotPaths` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -39,6 +40,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | | `stage-config-propagation` | pipeline sampling constraints、StageConfig serialization/runtime config | `VOMNI-CFG-1o` | | `stage-transport` | full-payload transport capability、topology projection、deploy/CLI override | `VOMNI-CFG-1p`，见 [stage transport capability](rules-stage-transport.md) | +| `model-reference-routing` | object-storage materialization、HF cache snapshot、name fallback | `CONF-7a`，见 [模型引用路由规则](rules-model-reference-routing.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `VOMNI-CFG-0a`, `VOMNI-CFG-0b` | ## 配置归一化与新老路径一致性 @@ -205,13 +207,6 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：沿用通用默认采样值，或仅凭经验调整敏感采样参数而不核对官方示例。 - 验收：解析 `vllm_omni/deploy/moss_tts_local.yaml`，断言上述参数与官方示例一致，并确认最终 stage 配置实际读取这些值。 ^[PR #6156] -### CONF-7a — 对象存储模型解析必须物化配置而保留原始 URI - -- 触发：父进程在 stage-specific `ModelConfig` 初始化前，需要从 `s3://`、`gs://` 或上游 `is_runai_obj_uri` 支持的对象存储 URI 推断模型类型、读取 HF 配置或解析 `model_index.json`。 -- 强制：通过 `is_runai_obj_uri` 将对象存储模型的 `*.model`、`*.py`、`*.json` 轻量文件按 URI 缓存一次到确定性的 `model_streamer/` 目录；`get_config`、`config.json` 和 `model_index.json` 的读取统一使用该本地目录，同时让各 stage 继续接收原始 URI 以保留 `model_weights` 的流式加载语义。 -- 禁止：把原始对象存储 URI 交给 Hugging Face 仓库解析；把临时 `model_streamer/` 路径传播成 stage 的模型身份；用完整 URI、bucket 或组织名参与模型类型 basename 匹配。 -- 验收：用 mock 覆盖对象存储 URI 的单次物化、配置驱动的 pipeline 选择、`model_index.json` 回退和 basename-only 匹配，并确认欺骗性 bucket 名不能劫持 pipeline；真实对象存储运行另行验证。^[PR #5036] - ### VOMNI-CFG-2a — 配置公开导出必须延迟加载以保持导入链无环 - 触发：修改 `vllm_omni.config` 的公开导出，或新增会经 pipeline registry、模型 pipeline 与 diffusion 数据模块回环的配置导入路径。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1ed5a2faa..85684e653 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f90e992d` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 7ba5e8a1` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6d31422a4..922b39ab5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f90e992d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7ba5e8a1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c9954b8d6..cc83411e8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f90e992d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7ba5e8a1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index be09a492c..d7e377890 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f90e992d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 7ba5e8a1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b9b29c8b2..0e6670c2c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f90e992d` 验证存在 +- 源码校验：以上路径均已在 `main @ 7ba5e8a1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md index 4586f6cc5..988519f7b 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, "PR #5036"] +sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, "PR #5036", "PR #6624"] confidence: high --- @@ -38,6 +38,5 @@ confidence: high - 触发：修改 `omni_snapshot_download()` 或入口侧模型解析，使其处理 Run:AI 对象存储 URI、HF repo id 和本地模型路径。 - 强制：使用上游 `is_runai_obj_uri` 作为唯一对象存储判断，在 Hugging Face/ModelScope 预下载前原样返回对象存储 URI；本地路径与普通 HF repo id 保持既有分支，并将后续配置读取交给配置层的本地物化逻辑。 -- 禁止：维护与 vLLM streamer 分离的本地 scheme allowlist；将对象存储 URI 当作 HF repo id 下载；让 bucket 或组织路径段参与 basename 模型匹配。 -- 验收：参数化 mock 测试覆盖 `s3://`、`gs://` 及上游支持的 `az://`，断言 HF 下载未调用且返回值未改变；同时覆盖本地路径、HF repo id 和欺骗性 bucket/basename 匹配，确认入口语义没有回归。^[PR #5036] - +- 禁止：维护与 vLLM streamer 分离的本地 scheme allowlist；将对象存储 URI 当作 HF repo id 下载；让 bucket 或组织路径段参与模型名称匹配。 +- 验收：参数化 mock 测试覆盖 `s3://`、`gs://` 及上游支持的 `az://`，断言 HF 下载未调用且返回值未改变；同时覆盖本地路径、HF repo id、有效 cache snapshot 的 repo-name recovery 与欺骗性 bucket/组织段匹配，确认入口语义没有回归。^[PR #5036] ^[PR #6624] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4b2d0be67..ebb212576 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f90e992d`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7ba5e8a1`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 76fd902fa7af177739be2b9ecaba48bdaa727223 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 20:33:06 +0800 Subject: [PATCH 314/465] knowledge(vllm-omni): sync 7439ed41 (PR #6642) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../configuration/rules-model-reference-routing.md | 10 +++++----- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-upstream-compat.md | 4 ++-- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a621c1707..8934cd558 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f90e992dc1dd45ed5cc3c536ee39567ef3f53470 + previous_audited_sha: 7ba5e8a146b41f7467caefa1795ecf6842b0077c audited_ref: main - audited_sha: 7ba5e8a146b41f7467caefa1795ecf6842b0077c + audited_sha: 7439ed41c27d6e817843cff7704d2068c5214ec6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 5d6f14f23..f87b78a8e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -7ba5e8a146b41f7467caefa1795ecf6842b0077c`. The canonical machine baseline +7439ed41c27d6e817843cff7704d2068c5214ec6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5cf4d1dad..80ae78a6b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7ba5e8a1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7439ed41` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-model-reference-routing.md b/knowledge/repos/vllm-omni/components/configuration/rules-model-reference-routing.md index e20eafc4c..76c7833c4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules-model-reference-routing.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules-model-reference-routing.md @@ -4,18 +4,18 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: [vllm_omni/config/config_factory.py, vllm_omni/config/pipeline_registry.py, tests/config/test_config_factory.py, "PR #5036", "PR #6624"] +sources: [vllm_omni/config/config_factory.py, vllm_omni/config/pipeline_registry.py, tests/config/test_config_factory.py, "PR #5036", "PR #6624", "PR #6642"] --- # 模型引用路由规则 ## Direct 代码快速入口 -- **CONF-7a — 模型引用解析。** 从 `config_factory.py::{_materialize_object_storage_configs,_name_match_candidate,StageConfigFactory._try_infer_model_type}` 直达 `pipeline_registry.py::OMNI_PIPELINES`；cache-shape 回归从 `tests/config/test_config_factory.py::TestNameMatchCandidateSnapshotPaths` 开始。 +- **CONF-7a — 模型引用解析。** 从 `config_factory.py::{_materialize_object_storage_configs,_name_match_candidate,StageConfigFactory._try_infer_model_type}` 直达 `pipeline_registry.py::OMNI_PIPELINES`；当前 name-fallback 的回归入口是 `tests/config/test_config_factory.py::TestObjectStorageConfigResolution`。 ### CONF-7a — 模型引用解析必须物化对象存储配置，并只从受控名称组件匹配 - 触发：父进程在 stage-specific `ModelConfig` 初始化前，需要从 `s3://`、`gs://` 或上游 `is_runai_obj_uri` 支持的对象存储 URI 推断模型类型、读取 HF 配置或解析 `model_index.json`。 -- 强制：通过 `is_runai_obj_uri` 将对象存储模型的 `*.model`、`*.py`、`*.json` 轻量文件按 URI 缓存一次到确定性的 `model_streamer/` 目录；`get_config`、`config.json` 和 `model_index.json` 的读取统一使用该本地目录，同时让各 stage 继续接收原始 URI 以保留 `model_weights` 的流式加载语义。最后的 name-based fallback 通常只比较末个路径组件；唯一例外是有效 HF hub cache snapshot 的尾形 `models----/snapshots/` 或 legacy `models--/snapshots/`，此时只恢复 repo 的 ``，不能让组织名参与匹配。 -- 禁止：把原始对象存储 URI 交给 Hugging Face 仓库解析；把临时 `model_streamer/` 路径传播成 stage 的模型身份；用完整 URI、bucket 或组织名参与模型类型名称匹配；把普通 `.../snapshots/` 或 malformed `models--...` 误认成 HF cache（它们必须继续取末个组件）。 -- 验收：用 mock 覆盖对象存储 URI 的单次物化、配置驱动的 pipeline 选择、`model_index.json` 回退和欺骗性 bucket 名；参数化覆盖 namespaced/legacy HF cache snapshot、trailing slash、普通路径与 malformed cache segment，断言只从允许的 repo-name 段恢复名称。空 `config.json` 的 snapshot 必须仍能由 `try_infer_model_type` 选中注册的 CosyVoice3 pipeline。PR #6624 的真实 H20 E2E 和 config-suite 结果由作者报告，未在本次知识审计环境复跑；其后续 review 还指出两个 consumer 注释仍称“basename-only”，这是文案漂移而非该提交已改动的行为。^[PR #5036] ^[PR #6624] +- 强制：通过 `is_runai_obj_uri` 将对象存储模型的 `*.model`、`*.py`、`*.json` 轻量文件按 URI 缓存一次到确定性的 `model_streamer/` 目录；`get_config`、`config.json` 和 `model_index.json` 的读取统一使用该本地目录，同时让各 stage 继续接收原始 URI 以保留 `model_weights` 的流式加载语义。当前 name-based fallback **只**比较去掉尾部 `/` 后的末个路径组件，不能让 bucket 或组织名参与匹配。 +- 禁止：把原始对象存储 URI 交给 Hugging Face 仓库解析；把临时 `model_streamer/` 路径传播成 stage 的模型身份；用完整 URI、bucket 或组织名参与模型类型名称匹配；也不要在这个目标版本把 `models----/snapshots/` 或 legacy `models--/snapshots/` 解码为 repo 名。 +- 验收：用 mock 覆盖对象存储 URI 的单次物化、配置驱动的 pipeline 选择、`model_index.json` 回退和欺骗性 bucket 名；空 `config.json` 时断言普通对象存储路径仍只以 basename 匹配。#6624 的 HF-cache repo-name recovery（含 CosyVoice3 空配置 snapshot）及其 `TestNameMatchCandidateSnapshotPaths` 回归测试已由 #6642 整体 revert；PR 正文只归因为 main Buildkite #14139 失败，公开 metadata 仅能确定其中 H100 entrypoint job hard-failed，不能从该记录归因到具体测试。故这不是当前实现合同。未解决后果是这类 resolved snapshot 会把 revision hash 当候选，可能无法推断模型类型并落入默认 diffusion 初始化；在恢复并验证更窄的修复前，使用显式 deploy pipeline 或一个以模型名结尾的模型引用。^[PR #5036] ^[PR #6624] ^[PR #6642] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 85684e653..c59cdd7e1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7ba5e8a1` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 7439ed41` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 922b39ab5..2fec242c6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7ba5e8a1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7439ed41` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cc83411e8..19a7908b6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7ba5e8a1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7439ed41` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d7e377890..870ca6537 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7ba5e8a1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 7439ed41` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0e6670c2c..c36baeda8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7ba5e8a1` 验证存在 +- 源码校验：以上路径均已在 `main @ 7439ed41` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md index 988519f7b..5c99ec712 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, "PR #5036", "PR #6624"] +sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, "PR #5036", "PR #6642"] confidence: high --- @@ -39,4 +39,4 @@ confidence: high - 触发：修改 `omni_snapshot_download()` 或入口侧模型解析，使其处理 Run:AI 对象存储 URI、HF repo id 和本地模型路径。 - 强制：使用上游 `is_runai_obj_uri` 作为唯一对象存储判断，在 Hugging Face/ModelScope 预下载前原样返回对象存储 URI；本地路径与普通 HF repo id 保持既有分支，并将后续配置读取交给配置层的本地物化逻辑。 - 禁止：维护与 vLLM streamer 分离的本地 scheme allowlist；将对象存储 URI 当作 HF repo id 下载；让 bucket 或组织路径段参与模型名称匹配。 -- 验收：参数化 mock 测试覆盖 `s3://`、`gs://` 及上游支持的 `az://`，断言 HF 下载未调用且返回值未改变；同时覆盖本地路径、HF repo id、有效 cache snapshot 的 repo-name recovery 与欺骗性 bucket/组织段匹配，确认入口语义没有回归。^[PR #5036] ^[PR #6624] +- 验收：参数化 mock 测试覆盖 `s3://`、`gs://` 及上游支持的 `az://`，断言 HF 下载未调用且返回值未改变；同时覆盖本地路径、HF repo id 和欺骗性 bucket/组织段匹配，确认入口语义没有回归。resolved HF cache snapshot 的 repo-name recovery 已随 #6642 的 revert 不再是当前合同。^[PR #5036] ^[PR #6642] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ebb212576..049e562d9 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7ba5e8a1`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7439ed41`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 1fb6dbef24320fcf8c7431c0165a6ee1258ecc0a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 20:44:33 +0800 Subject: [PATCH 315/465] knowledge(vllm-omni): sync 59cf3add (PR #6550) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules-lora.md | 11 ++++++----- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- knowledge/repos/vllm-omni/models/minimax-h3/_index.md | 2 +- .../vllm-omni/models/minimax-h3/rules-cache-task.md | 9 +++++---- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 2 +- 13 files changed, 23 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8934cd558..c8a8dff0a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7ba5e8a146b41f7467caefa1795ecf6842b0077c + previous_audited_sha: 7439ed41c27d6e817843cff7704d2068c5214ec6 audited_ref: main - audited_sha: 7439ed41c27d6e817843cff7704d2068c5214ec6 + audited_sha: 59cf3add400be1744f308f497bd31dcceb7be262 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f87b78a8e..bef852164 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -7439ed41c27d6e817843cff7704d2068c5214ec6`. The canonical machine baseline +59cf3add400be1744f308f497bd31dcceb7be262`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 80ae78a6b..6a9d62d35 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7439ed41` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 59cf3add` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c59cdd7e1..561f4305f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7439ed41` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 59cf3add` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md index d4af11132..0be6c4a88 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070", "PR #6476", vllm_omni/diffusion/models/minimax_h3/lora.py, tests/diffusion/lora/test_lora_manager.py] +sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/lora/layers/base_linear.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/diffusion/lora/test_lora_manager.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070", "PR #6476", "PR #6550", vllm_omni/diffusion/models/minimax_h3/lora.py] confidence: high --- @@ -20,7 +20,7 @@ confidence: high | `--lora-backend`、startup path/scale、deploy/CLI projection | `DIFF-2l` | `entrypoints/cli/serve.py` → `async_omni_engine.py`/stage config → `OmniDiffusionConfig` → `DiffusionWorker.init_lora_manager` | | distilled merge、key conversion、packed QKV、alpha、unload | `DIFF-2m` | `diffusion/lora/loader.py` → pipeline mixin → transformer parameter | | Wan dual transformer、high/low-noise file order、partial load | `DIFF-2n` | `WanLoraLoaderMixin` → `get_transformer_from_pipeline(name)` → `transformer`/`transformer_2` | -| legacy dynamic adapter hook、packed/stacked binding、activation rollback | `DIFF-2x` | `DiffusionLoRAManager::{_load_adapter,_bind_adapter_weights,_activate_adapter}` → model pipeline hook | +| legacy dynamic adapter hook、packed/stacked binding、DLO sidecar residency、activation rollback | `DIFF-2x` | `DiffusionLoRAManager::{_load_adapter,_bind_adapter_weights,_activate_adapter}` → model pipeline hook | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| @@ -93,8 +93,9 @@ confidence: high ## DIFF-2x — legacy dynamic manager 的模型专用 loader 与 activation 必须 fail-closed 且事务化 -- 触发：修改 `DiffusionLoRAManager` 的 request-time adapter load/bind、模型专用 loader hook 或 packed/stacked target。 +- 触发：修改 `DiffusionLoRAManager` 的 request-time adapter load/bind、模型专用 loader hook、packed/stacked target，或 DLO 下 dynamic sidecar 的 device placement/reallocation。 - 强制：先让 pipeline 的可选 loader 识别并返回其模型拥有的 `(LoRAModel, PEFTHelper)`，仅在其返回 `None` 时走既有 generic PEFT fallback；专用 loader 的 artifact、target 和全局 A/B shape 必须在 wrapper mutation 前完成校验。manager 必须记录实际绑定的 logical weights，模型 validator 证明完整覆盖；packed QKV/FFN 等布局由模型明确声明或转换，不由共享 manager 猜测。 - 强制：activation fast path 只可在 active ID 与 scale 都匹配时跳过。首次 mutation 前清除 active ID；任一 `set_lora()` 或 binding validator 异常都 reset 所有 wrapper slot 并保持 inactive，不能保留混合权重或陈旧 fast-path 状态。 -- 禁止：将已识别的专用 artifact 悄悄退回 generic loader；按名称而非真实绑定确认成功；将一个模型的 packed layout/scale 规则泛化给所有 diffusion 模型；失败后继续声明旧 adapter active。 -- 验收：CPU regressions 覆盖 model hook/fallback、binding completeness、mid-loop failure 后 reset 与旧 adapter retry；模型 integration 另覆盖 artifact identity、target/shape rejection、packed QKV/FFN binding 和实际 request sampling contract。^[PR #6476] +- 强制：DLO owns base-weight streaming only。DLO-enabled manager 在 wrapper replacement 时必须要求可用的 resident-buffer protocol、将 request-switchable A/B sidecars 放在 compute device，且 wrapper 因更大 rank 重新分配后必须重放该 device placement；缺 protocol fail closed。非 DLO path 不得获得此 placement side effect。 +- 禁止：将已识别的专用 artifact 悄悄退回 generic loader；按名称而非真实绑定确认成功；将一个模型的 packed layout/scale 规则泛化给所有 diffusion 模型；失败后继续声明旧 adapter active；把 sidecars 加入 DLO host shards/collective，或靠每 block transfer 伪装为 DLO support。 +- 验收：CPU regressions 覆盖 model hook/fallback、binding completeness、mid-loop failure 后 reset 与旧 adapter retry；DLO mock 覆盖 replacement 的 compute-device placement、缺 resident protocol 的 fail-closed 以及 rank-driven reallocation 后 A/B buffer 仍在该 device。模型 integration 另覆盖 artifact identity、target/shape rejection、packed QKV/FFN binding 和实际 request sampling contract。^[PR #6476] ^[PR #6550] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2fec242c6..30bd64b39 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7439ed41` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 59cf3add` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 19a7908b6..10b7bd23e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7439ed41` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 59cf3add` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 870ca6537..228c14bb2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7439ed41` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 59cf3add` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c36baeda8..0692bc64b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7439ed41` 验证存在 +- 源码校验：以上路径均已在 `main @ 59cf3add` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 049e562d9..ad4edfac5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7439ed41`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 59cf3add`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 473335387..8d93c139f 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -74,7 +74,7 @@ confidence: high 隔离见 MMH3-2h；它不替换 H3 的自定义 solver。 - step execution 是 H3 的模型专有 stateful path：只在 resolved attention backend 能隔离多个 packed documents 时合并 request；否则仍逐 request 前向。request state、rank-0 preparation、Cache-DiT/DLO/multi-output 排除与有界验证证据见 MMH3-2k，不能由 generic runner guard 推断 multi-rank failure synchronization。^[PR #5810] - LightX2V Turbo v1.0 仅支持 legacy dynamic LoRA；精确 artifact mapping、packed QKV/FC1 - binding、active-request sampling 合同与 offload 排除项见 MMH3-2j；转换后的 mixed-rank + binding、active-request sampling 合同与 DLO-resident A/B sidecar 边界见 MMH3-2j；转换后的 mixed-rank native PEFT checkpoint 不在该支持声明内。 - datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；single-request producer-owned `PackedPaddingMetadata` 可替代 structural suffix mask，generic ragged/continuous batches仍按 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md index 5369f6831..715363d6a 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py] +sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", "PR #6550", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py] confidence: high --- @@ -156,11 +156,12 @@ Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期 ## MMH3-2j — Turbo LoRA 只接受精确发布 artifact，并与 H3 task、sampling 和 offload lifecycle 共同门禁 -- 触发：MiniMax-H3 LightX2V Turbo 的 dynamic LoRA loader、artifact conversion、request task/steps/flow shift，或与 CPU/layerwise/distributed-layerwise offload 的组合。 +- 触发：MiniMax-H3 LightX2V Turbo 的 dynamic LoRA loader、artifact conversion、request task/steps/flow shift，或与 CPU、standard layerwise、distributed layerwise offload (DLO) 的组合。 - 强制：仅识别声明文件名的 LightX2V Turbo v1.0 safetensors，且 metadata 必须为 `key_format=minimax-h3-diffusers`、rank/alpha `128/128`；任何未知 tensor、重复/不完整 A/B pair、unsupported target 或不精确的 global A/B shape 都在 binding 前失败。转换必须把 Diffusers token-refiner/main-block names 映射为 native H3 names，恢复 FC1 `[gate; up]` 行序为 model-owned packed weights，并通过 `stacked_params_mapping` 将独立 Q/K/V 接到 packed QKV。 - 强制：Turbo 仅在非零 scale 的实际 active recognized adapter 时限制请求；只支持 FL2VA/T2VA，要求五个 sigma points（四次 denoiser evaluation）、video `flow_shift=6` 与 audio `audio_flow_shift=3`。每次真实 load 都替换该 client ID 的 Turbo classification，避免 eviction 后同 ID generic adapter 被误分类。 -- 禁止：将 exact Turbo artifact 的损坏 metadata 静默交给 generic fallback；将任意 H3 PEFT checkpoint、prefusion、multi-LoRA composition 或 Ref2VA 宣称为此功能支持；在 model-level CPU、layerwise 或 distributed-layerwise offload 下激活 Turbo，因为 legacy wrapper 的 LoRA tensors 不随 base parameter lifecycle 搬运。 -- 验收：覆盖 file/metadata/alpha/target/pair/shape rejection、QKV and FC1 packing、scale-zero/generic same-ID/Ref2VA lifecycle、steps/shifts errors 与三种 offload rejection；真实 artifact evidence 必须另绑定 exact checkpoint、task、topology、sampling 和 video/audio output。转换后的 native mixed-rank PEFT artifact 不属于本 PR 的 supported contract；其 generic fallback rank/scale mismatch 是 post-merge follow-up，不可用本页的 Turbo success 证明安全。^[PR #6476] +- 强制：DLO + active Turbo 可用时，DLO 只 stream base blocks；manager 注入的 request-switchable A/B sidecars 必须留在 compute device，且任何因更大 LoRA rank 触发的 wrapper reallocation 后仍重放该 placement。固定 sidecar HBM 是此组合的预算，不得把它算作 base-block DLO lifecycle 或 collective payload。 +- 禁止：将 exact Turbo artifact 的损坏 metadata 静默交给 generic fallback；将任意 H3 PEFT checkpoint、prefusion、multi-LoRA composition 或 Ref2VA 宣称为此功能支持；在 model-level CPU 或 standard layerwise offload 下激活 Turbo。不得为动态 sidecar 重建 DLO host shards、随每个 block/denoise step stream A/B，或让 LoRA 进入 DLO AllGather；没有 active LoRA 的 DLO 不得进入此 loader path。现有 HWR eligibility 仍拒绝 `lora_path`，AllGather 也不走该 BF16 HWR path，不能将 DLO+Turbo evidence 扩展为 HWR support。 +- 验收：覆盖 file/metadata/alpha/target/pair/shape rejection、QKV and FC1 packing、scale-zero/generic same-ID/Ref2VA lifecycle、steps/shifts errors、CPU/standard-layerwise rejection，以及 DLO admission、resident A/B device placement 和 rank-driven reallocation 后的 placement；真实 artifact evidence 必须另绑定 exact checkpoint、task、topology、sampling 和 video/audio output。DP validation 至少以同 wave 的并发 request 覆盖所声称 DP topology；DLO no-AllGather/AllGather evidence 只证明对应 base-weight transport，sidecars 不参与 collective。PR #6550 报告的 0.911 GiB/TP2-rank 与 1.355 GiB/TP1-rank 只绑定 rank-128、312-target 的该 Turbo artifact，不是任意 adapter 的容量保证。转换后的 native mixed-rank PEFT artifact 不属于本 PR 的 supported contract；其 generic fallback rank/scale mismatch 是 post-merge follow-up，不可用本页的 Turbo success 证明安全。^[PR #6476] ^[PR #6550] ## MMH3-2k — H3 step execution 必须保持请求状态、attention 文档和 rank-0 prepare 隔离 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index d03a323e4..144f377ba 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -33,7 +33,7 @@ confidence: high | force-refresh hint、once/repeat、reinstall key | [MMH3-2f](rules-cache-task.md#mmh3-2f-force-refresh-hint-属于-active-profile-identity) | H3 `extra_args` validation → immutable cache config → installation key/refresh context | | TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | [MMH3-2g](rules-cache-task.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state) | custom enabler → H3 extractor → module-resident hook state / per-generation reset | | distilled、DMD2、`base_schedule`、4-step、sigma boundary | [MMH3-2h](rules-cache-task.md#mmh3-2h-distilled-sigma-schedule-按-partition-所有且以区间计步) | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | -| LightX2V Turbo、legacy dynamic LoRA、packed QKV/FC1、five sigma points | [MMH3-2j](rules-cache-task.md#mmh3-2j-turbo-lora-只接受精确发布-artifact并与-h3-tasksampling-和-offload-lifecycle-共同门禁) | `lora.py::load_minimax_h3_turbo_lora` → pipeline hook/sampling validation → `DiffusionLoRAManager` | +| LightX2V Turbo、legacy dynamic LoRA、packed QKV/FC1、five sigma points、DLO resident A/B sidecars | [MMH3-2j](rules-cache-task.md#mmh3-2j-turbo-lora-只接受精确发布-artifact并与-h3-tasksampling-和-offload-lifecycle-共同门禁) | `lora.py::load_minimax_h3_turbo_lora` → pipeline hook/sampling validation → `DiffusionLoRAManager` | | step execution、continuous batching、multi-document packing、rank-0 prepare failure | [MMH3-2k](rules-cache-task.md#mmh3-2k-h3-step-execution-必须保持请求状态attention-文档和-rank-0-prepare-隔离) | `pipeline_minimax_h3.py::{prepare_encode,denoise_step,step_scheduler}` → `batched_packing.py` → runner | | DLO、TP-local、resident layers、allocator cache、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → `BoundedAllocatorCache`/`PinnedModuleStager` → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | From 63d5c31fc02608345e97bea328e0b4a3d9b0cb0a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 20:51:55 +0800 Subject: [PATCH 316/465] knowledge(vllm-omni): sync 816335cb (PR #6559) --- adapters/vllm_omni/release_baseline.yaml | 8 ++++---- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++++-- .../vllm-omni/models/mimo-audio/_index.md | 19 +++++++++++++------ 10 files changed, 30 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c8a8dff0a..aa41b8b7e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7439ed41c27d6e817843cff7704d2068c5214ec6 + previous_audited_sha: 59cf3add400be1744f308f497bd31dcceb7be262 audited_ref: main - audited_sha: 59cf3add400be1744f308f497bd31dcceb7be262 + audited_sha: 816335cb46191287e1bc1d734b8a496ae2e00da2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -19,8 +19,8 @@ inventories: count: 55 sha256: 9d5033c9a7ffd7ff636e32db0f9be2a549b7c16b034e13817e280f19a9e16769 deploy_yamls: - count: 83 - sha256: a229dcbee310f9ffe511ef103e09f278e626761a61abe658f866aa0a52564b85 + count: 84 + sha256: 2d8a2dcff97801aa87d08b4fbbdd6426bbe4e1c54fc767673b23d8b5ca4cb4e1 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index bef852164..8d4fd2797 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -59cf3add400be1744f308f497bd31dcceb7be262`. The canonical machine baseline +816335cb46191287e1bc1d734b8a496ae2e00da2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6a9d62d35..e3169aaac 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 59cf3add` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 816335cb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 561f4305f..7f8916537 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 59cf3add` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 816335cb` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 30bd64b39..6fc0c34e2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 59cf3add` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 816335cb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 10b7bd23e..82ec3bd73 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 59cf3add` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 816335cb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 228c14bb2..b65ab8f64 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 59cf3add` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 816335cb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0692bc64b..89792d356 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 59cf3add` 验证存在 +- 源码校验：以上路径均已在 `main @ 816335cb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ad4edfac5..e273869c5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 59cf3add`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 816335cb`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -38,7 +38,7 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | | Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 40 个模型族目录 | | Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 55 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 83 份 | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 84 份 | 对比上一审计快照（`39c16d75`）：AR 架构 88→81，diffusion pipeline 65→64， OMNI_PIPELINES 57→55，deploy 86→84。PR #6353 删除 AudioX 的 diffusion registry、 @@ -56,6 +56,10 @@ body 明确扩大为完整 first-party 删除。^[PR #6357] ^[Issue #5996] PR #6619 删除独立的 `minicpmo_4_5_duplex.yaml`，将 duplex 配置并入全部 shipping MiniCPM-o profile，因此 deploy inventory 84→83；这是 profile 合并，不是 MiniCPM-o 能力删除。^[PR #6619] +PR #6559 增加 `mimo_audio_5090d.yaml`，使 deploy inventory 83→84；这是 MiMo-Audio 的 +社区 1× RTX 5090/5090D 32 GB recipe profile，不增加 registry 架构或 pipeline key，且不构成 +其他 Blackwell SKU、online serving、性能或音质的支持证明。^[PR #6559] + PR #4820 将 `WanDMDPipeline` 注册为 Wan2.2 的同实现入口，因此 diffusion pipeline 计数回升至 65；它不是新的模型族或新的 `OMNI_PIPELINES` key。^[PR #4820] diff --git a/knowledge/repos/vllm-omni/models/mimo-audio/_index.md b/knowledge/repos/vllm-omni/models/mimo-audio/_index.md index 611fb0b55..01a109686 100644 --- a/knowledge/repos/vllm-omni/models/mimo-audio/_index.md +++ b/knowledge/repos/vllm-omni/models/mimo-audio/_index.md @@ -1,15 +1,15 @@ --- title: "MiMo-Audio（融合 thinker+talker 单 AR stage 语音模型）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-04 type: index tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/mimo_audio/, vllm_omni/deploy/mimo_audio.yaml, vllm_omni/model_executor/stage_input_processors/mimo_audio.py] +sources: ["PR #6559", recipes/XiaomiMiMo/MiMo-Audio.md, vllm_omni/model_executor/models/mimo_audio/, vllm_omni/deploy/mimo_audio.yaml, vllm_omni/deploy/mimo_audio_5090d.yaml, vllm_omni/model_executor/stage_input_processors/mimo_audio.py] --- # MiMo-Audio -以下事实在 `main @ 5d44868e` 复核。 +以下运行时事实在 `main @ 5d44868e` 复核；recipe/deploy 增量在 `main @ 816335cb` 复核。 ## 名称与范围 @@ -45,9 +45,16 @@ sources: [vllm_omni/model_executor/models/mimo_audio/, vllm_omni/deploy/mimo_aud ## 配置与 checkpoint 差异 -- `mimo_audio.yaml`（唯一 deploy）：`async_chunk: true` 单卡侧写;头注给出 - legacy 双卡 sync 模式的 `--stage-overrides` 用法;连接器 - `codec_chunk_frames 30` / `codec_left_context_frames 40`。 +- 默认 `mimo_audio.yaml` 是 `async_chunk: true` 单卡侧写；头注给出 legacy 双卡 + sync 模式的 `--stage-overrides` 用法，连接器为 `codec_chunk_frames 30` / + `codec_left_context_frames 40`。PR #6559 另加模型专用 + `mimo_audio_5090d.yaml`：同一 GPU 上 stage 0/1 的 `0.78`/`0.12` 显存比例、 + `TRITON_ATTN`、eager、无 prefix cache 与 disabled FlashInfer autotune。它是社区在 + 1× RTX 5090D 32 GB 的离线 `tts_sft` 观察，不是通用 Blackwell、在线服务或所有 + MiMo-Audio task 的运行时支持声明。 +- 该 recipe 的 CPU tokenizer 省显存路径依赖 PR #6539（或等效 tokenizer-device + 修复）；目标 pin 没有合入该运行时修复。因此不能把 `MIMO_AUDIO_TOKENIZER_DEVICE=cpu` + 当作此版本无条件可用的默认能力，也不能从 recipe 的配置外推性能、音质或其他 SKU。 - 在线 serving 必须带 MiMo 自己的 `chat_template.jinja` （`examples/online_serving/mimo_audio/`）。 - 已知常量不一致（pin 上如实记录）：`MAX_CODE2WAV_TOKENS=18192` 与 stage-1 From 663711e3b1c5657b70d1aeb155a331291e5e186b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:01:00 +0800 Subject: [PATCH 317/465] knowledge(vllm-omni): sync 030fccb2 (PR #6555) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 8 +++++++- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- knowledge/repos/vllm-omni/models/minimax-h3/_index.md | 8 +++++++- .../vllm-omni/models/minimax-h3/rules-deployment.md | 8 +++++++- .../repos/vllm-omni/models/minimax-h3/rules-media.md | 9 +++++++-- 14 files changed, 39 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index aa41b8b7e..01002cd02 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 59cf3add400be1744f308f497bd31dcceb7be262 + previous_audited_sha: 816335cb46191287e1bc1d734b8a496ae2e00da2 audited_ref: main - audited_sha: 816335cb46191287e1bc1d734b8a496ae2e00da2 + audited_sha: 030fccb2bfdc9f7d1c323650b8301a4448cbf8a8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8d4fd2797..dd9e7cee4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -816335cb46191287e1bc1d734b8a496ae2e00da2`. The canonical machine baseline +030fccb2bfdc9f7d1c323650b8301a4448cbf8a8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 47f69c7e8..fb284c37b 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -68,6 +68,6 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie | 遇到什么 | 查看哪里 | |---|---| -| 审查硬件 lane、回归 fence、CI 工具供应链或 xdist/shared fixture | [CI rules](rules.md) | +| 审查硬件 lane（含 MiniMax-H3 DLO DP2 ready smoke 的证据边界）、回归 fence、CI 工具供应链或 xdist/shared fixture | [CI rules](rules.md) | | 查看仓库特有 CI 陷阱 | [CI guides](guides/_index.md) | | 调查历史 CI 失败 | [CI incidents](incidents/_index.md) | diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 43b5455d5..3b9dab06d 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] confidence: high --- @@ -48,6 +48,12 @@ MiniCPM-o online suite 的具体矩阵必须从 server args 读取：core 只跑 collection 意图；没有实际 runtime 结果时，不能证明目标硬件行为。模型专属 fixture 合同见 [MCPMO-5b](../models/minicpm-o-4-5/rules.md#mcpmo-5b-online-serving-ci-必须显式区分-chunk-模式与-duplex-fixture-语义)。^[PR #6056] +MiniMax-H3 的 DLO DP2 ready smoke 同样必须把 YAML 注册和实际 runtime 证据分开：`h100_2`、 +selector、90-minute timeout 与 explicit dependencies 只证明该 job 的配置。用例以 TP=1/DP=2 +的 DLO server 发起两条并发 T2VA sync request，并检查 MP4 的 geometry/FPS 和非空可解码 audio； +PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload 的作者报告，不能 +提升为 Buildkite H100 pass、通用并发可靠性或性能 gate。^[PR #6555] + ## OMNI-CI-1b — 回归 fence 必须锚定可观察合同且非空转 - 触发：新增 bug regression、API guard、route census 或 middleware fence。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e3169aaac..bfbe95478 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 816335cb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 030fccb2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7f8916537..f06e75ef7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 816335cb` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 030fccb2` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6fc0c34e2..c28934db7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 816335cb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 030fccb2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 82ec3bd73..33734d69e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 816335cb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 030fccb2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b65ab8f64..4f95d8b66 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 816335cb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 030fccb2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 89792d356..571a8332e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 816335cb` 验证存在 +- 源码校验：以上路径均已在 `main @ 030fccb2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e273869c5..c720976c2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 816335cb`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 030fccb2`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 8d93c139f..876f93f87 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", .buildkite/cuda/test-nightly.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] confidence: high --- @@ -112,6 +112,12 @@ T2VA 因当前严格 gate 未达标被移出；这表示“无 retained pixel-le 它只验证单分区启动，不能外推 combined 双 DiT 的容量或性能。共享 offloader、并行和请求合同分别归 [Diffusion](../../components/diffusion/_index.md)、 [Configuration](../../components/configuration/_index.md) 和 [Serving](../../components/serving/_index.md)。 +另有 ready L2 smoke 使用 TP=1/DP=2 的 distributed layerwise offload、two concurrent +`/v1/videos/sync` T2VA requests、4 steps 和 1344×768/24 FPS；它同时断言 MP4 video 与 +nonempty decodable audio。该 YAML job 配置为两张 H100，但 PR 仅报告 local two-card B300 +execution，不能代替 retained accuracy gate、Buildkite H100 result 或其他 parallel topology +的支持证据。^[PR #6555] + ## 审查入口 H3 input matrix/media ingress，以及 text-encoder completeness、online FP8 的 component namespace、loader 顺序、joint quality 与 offload 边界见 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index bf1ddb0ed..c4209cfb6 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py", "PR #6072", "PR #6526", tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/offloader/test_module_residency.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", "PR #6555", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py", "PR #6072", "PR #6526", tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/offloader/test_module_residency.py] confidence: high --- @@ -234,6 +234,12 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule - 禁止：让 backend 自行决定 checkpoint compatibility 或混合 DiT/非 DiT source 后只跳过部分 materialization；把 TP>1 ordinary-loader fallback、HSDP、online quantization 或其他 fail-closed 路径宣称具有 node-shared mmap 节省；把 resident layers 增加描述成减少 rank-local pinned host master。 - 验收：覆盖 dedicated/mixed source、strict non-DiT loading、AllGather shard 与 handle release、rank-local source retention、双 staging slot 上界、group size one 无 collective、TP>1 ordinary-loader fallback 及 unsupported-platform 不能静默丢弃 plan。PR 的 DP=2/TP=1 no-AllGather 与四 GPU topology smoke 只证明报告的 H3 配置和输出/资源观察；TP=2 行证明的是 ordinary-loader fallback，不是 shared-mmap memory guarantee。^[PR #6213] + PR #6555 adds a separate ready smoke for TP=1/DP=2 DLO with two concurrent T2VA + requests, but its server arguments do not include `--dlo-no-use-allgather`; do not use + it to widen this rule's no-AllGather/direct-mmap claim. Its reported local B300 pass is + limited runtime evidence for that exact request workload, while the H100_2 YAML entry + remains collection/configuration intent until the lane result is available. ^[PR #6555] + ## MMH3-3k — H3 VAE decoder 的 tile 不足 rank 时必须进入 rank-local decode - 触发：MiniMax-H3 修改 decoder tile 切分、`vae_patch_parallel_size`、parallel state，或低分辨率 preview/smoke 测试可能产生少于 tile group rank 数的 decoder tile。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md index 900a3a613..101e531fe 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5752", "PR #5829", "PR #5978", .buildkite/cuda/test-nightly.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064"] +sources: ["PR #5752", "PR #5829", "PR #5978", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064"] confidence: high --- @@ -41,6 +41,12 @@ confidence: high 的生成时域上界、frame-count/FPS 一致性与非零 rank 非正 duration 在 collective 前 fail-fast。^[PR #5752] ^[PR #5978] +- Online-serving T2VA smoke may use `assert_video_valid` for MP4 geometry/FPS, but that + helper does not establish audio presence. The H3 DP2 smoke must additionally open the + returned MP4, require an audio stream, and decode a frame with positive sample count. + This checks one response's muxed-media contract only; it is not a pixel-quality or + audio-quality gate. ^[PR #6555] + ## MMH3-2b — media ingress 在解码前限界，request 错误保持 HTTP 400 - 触发：typed `image_reference`/`video_reference`/`audio_reference`、multipart @@ -81,4 +87,3 @@ confidence: high - 强制：`_probe_video` 优先使用容器提供的 `nb_read_frames`/`nb_frames`，仅在元数据缺失时追加 `-count_frames` 扫描；Qwen 抽样必须将请求的 frame indices 交给一次 ffmpeg raw-video pipe，通过 `select` 只解码这些帧，完整回退也复用同一 ffmpeg RGB24 路径。解码前校验 frame count、宽高和 indices，必须在 reshape 前校验输出字节数等于 `frame_count × width × height × 3`。 - 禁止：为每个采样帧启动一次 ffmpeg、无条件对高码率 prepared stream 做完整扫描、在 decord 缺失时重新使用逐帧 PyAV 全量解码，或对不完整 raw-video 输出静默 reshape；不得把该 CPU 解码优化扩展成 MUSA/NPU 专用路径或通用 decoder 性能保证。 - 验收：contract tests 覆盖 selective/full ffmpeg command、`select` indices、decord 缺失回退、容器帧数直读与缺失时的 count-frames fallback、非法 metadata 和 incomplete output；在固定 prepared video 上将 selective 与 full RGB24 数组逐像素对照 PyAV reference，并分别核对 sampled frame 数、尺寸和 timestamps。^[PR #6064] - From 0fa3c8be5199215791d4f01d3f85c3a289ba98e6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:06:26 +0800 Subject: [PATCH 318/465] knowledge(vllm-omni): sync 5ec576ab (PR #6564) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../serving/rules-engine-lifecycle.md | 21 +++++++++++++++++-- .../vllm-omni/components/serving/rules.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 30 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 01002cd02..47593e7b8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 816335cb46191287e1bc1d734b8a496ae2e00da2 + previous_audited_sha: 030fccb2bfdc9f7d1c323650b8301a4448cbf8a8 audited_ref: main - audited_sha: 030fccb2bfdc9f7d1c323650b8301a4448cbf8a8 + audited_sha: 5ec576ab4c241afa8b56bee729776ccf7a0eb5eb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index dd9e7cee4..ea3c74987 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -030fccb2bfdc9f7d1c323650b8301a4448cbf8a8`. The canonical machine baseline +5ec576ab4c241afa8b56bee729776ccf7a0eb5eb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bfbe95478..2e7f181b0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 030fccb2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5ec576ab` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f06e75ef7..feb0f2d7e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 030fccb2` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5ec576ab` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c28934db7..2c39a6a6f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 030fccb2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5ec576ab` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 33734d69e..eaf879ee0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 030fccb2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5ec576ab` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4f95d8b66..9d12ebd1c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 030fccb2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5ec576ab` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 571a8332e..da685596b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 030fccb2` 验证存在 +- 源码校验：以上路径均已在 `main @ 5ec576ab` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 0e9150c8b..c3fb469ed 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,13 +4,14 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py] confidence: high --- # Serving engine 生命周期规则 -`SERV-5a`–`SERV-5e` 与 `SERV-6a`–`SERV-6d`。触发条件与其余审查组见 +本页收纳 `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5p`、`SERV-6a`–`SERV-6f`、 +`SERV-8a` 与 `SERV-9a`。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口；请求输入侧的合同留在该页， 故障隔离见 [fault isolation 规则](rules-fault-isolation.md)。 @@ -179,6 +180,22 @@ confidence: high 同步。PR #4092 只声称既有测试通过，未提供可复现命令或该 proxy 回归；其要求 inline guard/comment 的 review thread 在合并时仍未 resolve，故这条验收尚未被该 PR 证明。^[PR #4092] +### SERV-5p — streaming final stage 的 raw terminal 必须闭合一次 output lifecycle + +- 触发：`async_chunk` streaming/realtime 的 final output stage 收到 raw `finish_reason`，或修改 + orchestrator raw/processed poll、terminal routing、request cleanup。 +- 强制：只把非 segment 的 session-level raw terminal 计入 final-output-stage completion；同一 poll + 必须先路由 processed outputs。若所有 final output stages 已完成、仍存在 request state、且没有 + processed terminal 完成 cleanup，则使用既有 terminal-empty output helper 发出一个 `finished=True` + 的最终 `OutputMessage`，然后走普通 request/CFG-parent cleanup。full-duplex session 不适用这个 + fallback。 +- 禁止：仅记录 raw completion 而让 Realtime generator 永远等不到 terminal message；在 processed + data 前发送 fallback；为普通 segment stop 或非-final stage 合成 terminal；用超时伪造完成，或在 + 已 cleanup 后再发第二个 terminal。 +- 验收：CPU 覆盖 raw-terminal-only、data-then-raw-terminal、processed-terminal-plus-raw-terminal + 三种同 poll 形状，分别断言完成、数据先于 terminal、exactly once 和 request state 清理；Qwen3-Omni + Realtime async-chunk 长音频 E2E 断言 `response.audio.done` 到达且音频有效。^[PR #6564] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 16f470d1e..6d21aaded 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -44,7 +44,7 @@ confidence: high | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | -| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5m`–`SERV-5n`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）, `SERV-5f` | +| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、streaming raw terminal、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5p`（见 [engine 生命周期规则](rules-engine-lifecycle.md)），`SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds | `SERV-4a`–`4h`, `SERV-4l`–`4p`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c720976c2..cec6a908b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 030fccb2`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5ec576ab`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 43f3383fe79e9fa0c2efcbdd85478c210e661cbc Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:12:34 +0800 Subject: [PATCH 319/465] knowledge(vllm-omni): sync d3c990dc (PR #6342) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../vllm-omni/models/ltx2/architecture.md | 23 +++++++++++++++++-- 10 files changed, 31 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 47593e7b8..79144b416 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 030fccb2bfdc9f7d1c323650b8301a4448cbf8a8 + previous_audited_sha: 5ec576ab4c241afa8b56bee729776ccf7a0eb5eb audited_ref: main - audited_sha: 5ec576ab4c241afa8b56bee729776ccf7a0eb5eb + audited_sha: d3c990dca8c3787ffdfe1d1424cfda9584d6bff9 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ea3c74987..a045c17c9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5ec576ab4c241afa8b56bee729776ccf7a0eb5eb`. The canonical machine baseline +d3c990dca8c3787ffdfe1d1424cfda9584d6bff9`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2e7f181b0..26981919a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5ec576ab` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d3c990dc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index feb0f2d7e..85d6210ec 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5ec576ab` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ d3c990dc` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2c39a6a6f..1655744c9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5ec576ab` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d3c990dc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index eaf879ee0..7eb38e239 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5ec576ab` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d3c990dc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9d12ebd1c..fe5f596a2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5ec576ab` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d3c990dc` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index da685596b..5c506d44b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5ec576ab` 验证存在 +- 源码校验：以上路径均已在 `main @ d3c990dc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index cec6a908b..e269919b5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5ec576ab`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d3c990dc`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/ltx2/architecture.md b/knowledge/repos/vllm-omni/models/ltx2/architecture.md index b2464d49a..a7562575d 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/architecture.md +++ b/knowledge/repos/vllm-omni/models/ltx2/architecture.md @@ -1,10 +1,10 @@ --- title: "LTX-2/2.3 模型架构与证据索引" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-04 type: architecture tags: [vllm-omni, models, ltx2] -sources: [recipes/LTX/LTX-2.md, vllm_omni/diffusion/registry.py, "#4381", "#4464"] +sources: [recipes/LTX/LTX-2.md, vllm_omni/diffusion/registry.py, "#4381", "#4464", "PR #6342", vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_vae.py, tests/diffusion/models/ltx2/test_ltx2_vocoder_cuda.py, tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py, tests/e2e/accuracy/ltx/test_ltx_official_similarity.py] --- # LTX-2/2.3 模型架构与证据索引 @@ -30,6 +30,25 @@ sources: [recipes/LTX/LTX-2.md, vllm_omni/diffusion/registry.py, "#4381", "#4464 `async_omni_engine.py` 的默认 diffusion stage 兜底），deploy 语义见 [Config 组件](../../components/configuration/architecture.md)。 +## 音频 parity 与初始化合同 + +- 自 `main @ d3c990dc` 起，带 `bwe_generator` 的 LTX BWE vocoder 在 CUDA 路径使用 FP32 + 输入及 FP32 autocast 执行长卷积栈，随后将波形转换回 decode 输入 dtype；不带该属性的 + 基础 vocoder 保持原生调用。MPS 路径临时将 BWE 模块转为 FP32，并在调用后恢复原参数 + dtype。合并代码以真实 CUDA BF16 `Conv1d` 回归验证中间输出为 FP32，但关于 CUDA + FP32-autocast 可移植性的 P1 thread 在合并时仍未 resolve；该证据不得外推到未验证 backend。 +- `latent_upsampler` 必须在显式 CPU device context 中构造、再加载权重；这是 + rational-resampler 的 Long-tensor 初始化合同，避免 CUDA default-device 下不支持的 + Long `addmm`，不改变已加载权重的 dtype 或运行时 placement。 +- 官方相似度守卫对保留的 LTX-2.0/2.3/2.5 case 使用同一 profile：video SSIM mean/min + ≥0.99、PSNR mean ≥40 dB、audio relative-L2 ≤0.10、audio cosine ≥0.99。该 profile + 是精度回归门，而不是跨 checkpoint、硬件或配置的泛化质量声明。Omni runner 必须通过 + symlink overlay 暴露已经解析并 pin 的 upsampler/LoRA sidecar，防止两侧比较不同 artifact。 +- retained matrix 直接保留 LTX-2.5 Full one-stage，并以 LTX-2.0/2.3 two-stage 的 Stage 1 + 覆盖相应 denoising path；它删除了 LTX-2.0/2.3 的 direct one-stage case。关于这是否遗漏 + `(one_stage, version)` recipe/profile dispatch 的 P2 thread 在获批合并时仍未 resolve，因此 + 不得把 retained matrix 描述成对这两个版本 one-stage entry 的直接覆盖。^[PR #6342] + ## 已有证据索引（只链接，不复制正文） - 性能/profiling 陷阱：eager-trace 与 graph-benchmark 口径混同、mask-sync"优化" From 8c8f463c1534d969a55a96bc8ce12baaa54f30cb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:19:41 +0800 Subject: [PATCH 320/465] knowledge(vllm-omni): sync 8cfede43 (PR #5221) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../configuration/environment-variable-contract.md | 4 ++-- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 4 ++-- .../components/serving/rules-engine-lifecycle.md | 11 +++++++++-- knowledge/repos/vllm-omni/components/serving/rules.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 12 files changed, 23 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 79144b416..bedb94951 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5ec576ab4c241afa8b56bee729776ccf7a0eb5eb + previous_audited_sha: d3c990dca8c3787ffdfe1d1424cfda9584d6bff9 audited_ref: main - audited_sha: d3c990dca8c3787ffdfe1d1424cfda9584d6bff9 + audited_sha: 8cfede4309cb22048013393212a21fd233876ffa # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a045c17c9..06e85a0d8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d3c990dca8c3787ffdfe1d1424cfda9584d6bff9`. The canonical machine baseline +8cfede4309cb22048013393212a21fd233876ffa`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 26981919a..f2d365720 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d3c990dc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8cfede43` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md index 9a9c5a725..b1de7109b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md +++ b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: guide tags: [vllm-omni, config, environment] -sources: ["PR #6217", "PR #6631", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] +sources: ["PR #6217", "PR #6631", "PR #5221", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] --- # 环境变量合同与诊断边界 @@ -19,7 +19,7 @@ sources: ["PR #6217", "PR #6631", vllm_omni/config/environment_variable_inventor ## 已合并的合同 -- `environment_variable_inventory.py` 是**分类清单，不是值解析器**：实际 consumer 继续拥有默认值、解析、优先级和读取时机。`main @ a708ae5b` 的 `PUBLIC_OMNI` 审阅边界有 23 个名称；新公共 Omni 名称必须以 `VLLM_OMNI_` 开头，旧前缀仅为兼容保留。 +- `environment_variable_inventory.py` 是**分类清单，不是值解析器**：实际 consumer 继续拥有默认值、解析、优先级和读取时机。PR #5221 将 `VLLM_OMNI_EVENT_DRIVEN_ORCH` 加入 `PUBLIC_OMNI`（24 个名称）：只在 `Orchestrator` 构造时读取，默认 `0`；trim/lower 后仅 `1`、`true`、`yes`、`on` 开启，其他值保留 legacy loop。它同时选择 LLM orchestration 与 serving final-output drain，部署时必须在运行 orchestrator 的 stage-0 进程启动前设置；不是 request-level 开关，且不使 diffusion output 变为无轮询。新公共 Omni 名称必须以 `VLLM_OMNI_` 开头，旧前缀仅为兼容保留。^[PR #5221] - PR #6631 处理 #6217 合并后的 main 漂移：`VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` 是公开、实验性、仅环境变量的 float 秒数开关，默认 `600`；diffusion engine 在每次 `step_streaming` 请求路径读取它，非 float 或 `<=0` 时仅 warning-once 并回退默认值。它等待 async output，不是 `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` 的 full-payload input-coordinator 控制。`ROSVOT_SOURCE_DIR` 已随 SoulX-Singer 支持移除而从 model-specific `deprecate_remove` 清单删除。 - 公开设置没有全局的 env-vs-CLI/YAML/request 优先级。按每个 consumer 的合同核实：请求可变行为应属于 request schema，稳定 model/stage 行为应属于 typed config，环境变量只适合启动、平台集成、诊断或兼容回退。import/startup 时读取的值必须在启动 CLI/导入前设置；启动后的父 shell 变化不会更新既有 worker。 - stage 的 `env`（以及兼容的 `runtime.env`）仅在启动该 stage/child process 时临时应用：键和值会字符串化，child 可继承，随后恢复父进程旧环境。因此它不能追溯改变已在此前 import 缓存的值，也不应承载 request-varying 语义。日志和诊断只输出键，绝不输出 stage `env` 的值。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 85d6210ec..005442a96 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d3c990dc` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 8cfede43` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1655744c9..300c47c09 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d3c990dc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8cfede43` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7eb38e239..7bd934668 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d3c990dc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8cfede43` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index fe5f596a2..1b31ff16a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d3c990dc` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8cfede43` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5c506d44b..940eadb5c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d3c990dc` 验证存在 +- 源码校验：以上路径均已在 `main @ 8cfede43` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -32,5 +32,5 @@ sources: [] | upstream launcher shutdown、renderer warmup 与多 replica compile-cache 隔离 | [upstream 兼容规则](rules-upstream-compat.md) | | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | -| engine startup/shutdown 顺序、stage 生命周期、full-duplex 与 CFG companion | [engine 生命周期规则](rules-engine-lifecycle.md) | +| engine startup/shutdown、stage lifecycle、full-duplex、CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | | 公开请求字段的校验、来源冲突、alias/extras 归一与 consumer view | [请求输入合同](rules-request-input.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index c3fb469ed..1396cfcc5 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,13 +4,13 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5221", "Issue #4855", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, tests/engine/test_orchestrator_event_driven.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py] confidence: high --- # Serving engine 生命周期规则 -本页收纳 `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5p`、`SERV-6a`–`SERV-6f`、 +本页收纳 `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5q`、`SERV-6a`–`SERV-6f`、 `SERV-8a` 与 `SERV-9a`。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口；请求输入侧的合同留在该页， 故障隔离见 [fault isolation 规则](rules-fault-isolation.md)。 @@ -196,6 +196,13 @@ confidence: high 三种同 poll 形状，分别断言完成、数据先于 terminal、exactly once 和 request state 清理；Qwen3-Omni Realtime async-chunk 长音频 E2E 断言 `response.audio.done` 到达且音频有效。^[PR #6564] +### SERV-5q — 事件驱动编排只能改变唤醒机制并保持双路径合同 + +- 触发：修改 `VLLM_OMNI_EVENT_DRIVEN_ORCH`、orchestrator output loop、stage reader/poller、或 `AsyncOmni` final-output drain。 +- 强制：默认关闭时保留 legacy 1 ms poll loop；开启时每个 live LLM replica 的 reader 直接 await `get_output_async()`，再进入单一串行 dispatch，使 routing、output ordering、terminal cleanup、scheduler stats 与 per-replica `EngineDeadError` isolation 同 legacy path。reader 必须每 0.5 s 按 live membership 与 client object reconcile，client swap、eviction 或新注册不得遗失输出或留下 task。diffusion 维持 `get_diffusion_output_nowait()` poller，metrics-only sentinel 在 route 前吸收。final-output drain 使用同一 flag 的 queue blocking read，engine 缺少该方法时回退 legacy drain；1 s timeout 只用于 orchestrator liveness check，shutdown 必须关闭其 executor/queue 资源。 +- 禁止：把 "event-driven" 外推为所有 stage 均无 polling；从一个 loop 删除 metrics、raw-terminal 或 fault-isolation handling；在 busy queue 下只在 timeout reconcile，或让 reader failure 重启为重复 eviction；把两种 loop 的 orch-monitor iteration counts/`loop_active_pct` 直接比较。 +- 验收：以 flag guard 重跑 legacy LLM/diffusion, async-chunk, abort, shutdown, multi-replica, scheduler-stat 和 dead-replica parity scenarios；另测 accepted/unrecognized flag values、default legacy selection、client swap 后 reader reconcile、blocking drain 的 queued/late message、timeout/liveness 与 executor shutdown。性能测试另绑定同一 current-head build、硬件、workload、warmup 与多次 A/B；PR #5221 的 H20 数字在 fault-isolation rebuild 前测得，不能作为 current-head performance proof。两条文档 review 要求已进入最终代码，但对应 thread 合并时仍为 unresolved。^[PR #5221] ^[Issue #4855] + ### SERV-6a — full-duplex 首次 stage submit 必须预热 async-chunk topology - 触发：full-duplex stage port、双工会话、async-chunk 或 stage fence 发生变化。 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 6d21aaded..7dc5c40bb 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -44,7 +44,7 @@ confidence: high | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | -| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、streaming raw terminal、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5p`（见 [engine 生命周期规则](rules-engine-lifecycle.md)），`SERV-5f` | +| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、streaming raw terminal、event-driven orchestration、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5q`（见 [engine 生命周期规则](rules-engine-lifecycle.md)），`SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds | `SERV-4a`–`4h`, `SERV-4l`–`4p`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e269919b5..812d77d41 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d3c990dc`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8cfede43`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From ace518d6d2784c535e9559d1e7a8ba6a401fe845 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:25:26 +0800 Subject: [PATCH 321/465] knowledge(vllm-omni): sync 926e7fbf (PR #5769) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../repos/vllm-omni/models/glm-tts/_index.md | 26 ++++++++++++++++--- 10 files changed, 33 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bedb94951..038a1c14a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d3c990dca8c3787ffdfe1d1424cfda9584d6bff9 + previous_audited_sha: 8cfede4309cb22048013393212a21fd233876ffa audited_ref: main - audited_sha: 8cfede4309cb22048013393212a21fd233876ffa + audited_sha: 926e7fbfa2340d033016be79bc5b57dff9152a20 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 06e85a0d8..c6d2f844c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8cfede4309cb22048013393212a21fd233876ffa`. The canonical machine baseline +926e7fbfa2340d033016be79bc5b57dff9152a20`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index f2d365720..2e013026e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8cfede43` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 926e7fbf` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 005442a96..f1d564753 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8cfede43` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 926e7fbf` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 300c47c09..4ef49cac6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8cfede43` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 926e7fbf` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7bd934668..90021c3d0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8cfede43` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 926e7fbf` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1b31ff16a..19d3e682b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8cfede43` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 926e7fbf` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 940eadb5c..9dd80efe9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8cfede43` 验证存在 +- 源码校验：以上路径均已在 `main @ 926e7fbf` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 812d77d41..b48bd5869 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8cfede43`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 926e7fbf`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/glm-tts/_index.md b/knowledge/repos/vllm-omni/models/glm-tts/_index.md index 49002ec5d..79b4772e6 100644 --- a/knowledge/repos/vllm-omni/models/glm-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/glm-tts/_index.md @@ -1,15 +1,16 @@ --- title: "GLM-TTS（AR + flow-matching DiT 流式 TTS）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-04 type: index tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/glm_tts/, vllm_omni/deploy/glm_tts.yaml, vllm_omni/config/pipeline_registry.py] +sources: [vllm_omni/model_executor/models/glm_tts/, vllm_omni/deploy/glm_tts.yaml, vllm_omni/config/pipeline_registry.py, recipes/zai-org/GLM-TTS.md, "PR #5769"] --- # GLM-TTS -以下事实在 `main @ 5d44868e` 复核（源码派生页,尚无本模型的运行经验沉淀）。 +以下源码与 recipe 在 `main @ 926e7fbf` 复核。部署观测是社区 recipe 的有界证据， +不是通用硬件支持或性能保证。 ## 名称与范围 @@ -38,6 +39,25 @@ sources: [vllm_omni/model_executor/models/glm_tts/, vllm_omni/deploy/glm_tts.yam `hf_overrides`（`sample_method: ras` + top-k/p 窗口）;标注 1×A40 验证 （~16.6 GiB）。 +## 社区双 GPU 部署证据（非默认合同） + +- PR #5769 的作者在两张各报告 49,140 MiB 的**非标准 48 GB RTX 4090**上验证过该 + recipe；它不适用于或证明 stock 24 GB RTX 4090。该次环境为 Ubuntu 22.04.1、driver + 580.76.05、Python 3.12.13、PyTorch 2.11.0+cu130、CUDA 13.0、vLLM 0.26.0、 + vLLM-Omni `0.26.0rc2.dev5+g66a9b0c84` 与 FlashInfer 0.6.14。宿主 `nvcc` 为 + 11.8 时，作者将 `CUDA_HOME` 指向虚拟环境内的 CUDA 13 toolkit；这是该环境的 + 启动前提，不是 deploy YAML 的可移植默认值。 +- 该 recipe 保持 bundled `glm_tts.yaml` 为配置真源：默认 async-chunk，将可见 GPU + 设为 `0,1` 后以 `--stage-overrides '{"1":{"devices":"1"}}'` 把 stage 1 放到 GPU 1， + stage 0 留在 GPU 0；需要同步变体时只加 `--no-async-chunk`。`--omni`、 + `--trust-remote-code`、`--deploy-config vllm_omni/deploy/glm_tts.yaml` 与受限本地 + media path 都是该已报告命令的一部分。 +- 作者以同一中文 prompt/reference、并发 1、一次 warm-up 后三次测量，报告双卡相对 + 单卡的 async 非流式、async 流式和同步 wall time 分别降低 8.2%、11.4% 和 1.6%； + async 流式 TTFA 报告降低 6.0%。这些是单次受控运行的作者观测，不能外推到其他 + GPU、driver、workload 或当前 head。中英文 voice-cloning smoke 均报告为 24 kHz 单声道 + PCM WAV；复现时应先以 `ffprobe` 验证输出格式，再报告延迟或 RTF。^[PR #5769] + ## 什么时候查这里 - 审查 GLM-TTS 的 DiT CUDA-graph bucket、渐进分块流式或精度（fp32 stage 1） From 2c7f8df1e780b8aa5fa3c9894fc603f141adc7f6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:33:44 +0800 Subject: [PATCH 322/465] knowledge(vllm-omni): sync 91dd7fca (PR #6318) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../models/minicpm-o-4-5/rules-duplex.md | 21 ++++++++++++++++++- .../vllm-omni/models/minicpm-o-4-5/rules.md | 12 +---------- .../vllm-omni/models/personaplex/rules.md | 15 +++++++++++-- 12 files changed, 44 insertions(+), 24 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 038a1c14a..96f63f516 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8cfede4309cb22048013393212a21fd233876ffa + previous_audited_sha: 926e7fbfa2340d033016be79bc5b57dff9152a20 audited_ref: main - audited_sha: 926e7fbfa2340d033016be79bc5b57dff9152a20 + audited_sha: 91dd7fcaec7419cfc93586f2cc2dca1b09240a4e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c6d2f844c..ee5eafbf8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -926e7fbfa2340d033016be79bc5b57dff9152a20`. The canonical machine baseline +91dd7fcaec7419cfc93586f2cc2dca1b09240a4e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2e013026e..fb11770fa 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 926e7fbf` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 91dd7fca` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f1d564753..9274331ec 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 926e7fbf` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 91dd7fca` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4ef49cac6..af0421588 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 926e7fbf` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 91dd7fca` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 90021c3d0..7ab665c1e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 926e7fbf` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 91dd7fca` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 19d3e682b..9cb48c74c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 926e7fbf` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 91dd7fca` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9dd80efe9..9857e1842 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 926e7fbf` 验证存在 +- 源码校验：以上路径均已在 `main @ 91dd7fca` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b48bd5869..796d0a06b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 926e7fbf`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 91dd7fca`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md index 41f2cfc5c..5a77e49bb 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -4,12 +4,31 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6346", "PR #6458", "PR #6619", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, tests/config/test_config_factory.py, tests/entrypoints/openai_api/test_duplex_handler.py] +sources: ["PR #6318", "PR #6346", "PR #6458", "PR #6619", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, tests/config/test_config_factory.py, tests/entrypoints/openai_api/test_duplex_handler.py] confidence: high --- # MiniCPM-o 4.5 native duplex 规则 +## MCPMO-4b — native duplex session update 必须如实拒绝不可重建的上下文 + +- 触发：运行中更新 MiniCPM-o native duplex 的 `instructions`/native-mode。 +- 强制：MiniCPM-o 的 changed `instructions` 由 runtime adapter 返回 + `instructions_update_unsupported`，不能 ACK 成 `session.updated`；serving 层也在 native context + 已锁时拒绝 changed instructions。`native_context_locked` 仅在真实 native append 成功后设置， + buffered、deferred 或失败 append 不锁。若创建时已给出 `minicpmo45_native_duplex`，后续改变该值 + 返回 `native_duplex_mode_update_unsupported`。共享 helper 保持“值改变即错误”的 adapter 边界一致； + PersonaPlex 的对应 persona/voice 合同见 [PPLEX-1c](../personaplex/rules.md#pplex-1c-persona-与-voice-在-session-创建后不可变)。 +- 若以后改为允许任何会影响 `prefill_slots` 或 first-append context-token reservation 的更新，必须先 + 重算并验证派生 reservation，再 ACK 并原子提交；资源不足时不得留下部分 mutation。这是现有拒绝 + 行为所保留的安全边界，不是本提交新增的可变配置能力。 +- 禁止：在 Stage0 已用旧 context 初始化后 ACK 变更；buffer/defer/reject append 提前锁定；用 mode + flip 绕过锁。 +- 验收：覆盖 post-lock changed/unchanged instructions、短 buffer 后 update 再 commit与 mode flip； + 拒绝时不发 runtime `session.update` signal，而 commit 仍可 flush 已缓冲音频。重建 + first-append token/prefill reservation 后允许变更是 PR body + 提到的后续 GPU-validated 工作，非本提交已实现语义。 ^[PR #6318] + ## MCPMO-4c — native duplex Talker 必须按 generate_chunk 预算终止 - 触发：修改 MiniCPM-o native duplex Talker 的 `generate_chunk`、codec EOS 或请求级音频状态。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index f8a3c3442..bfa1f0e05 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -26,7 +26,7 @@ confidence: high | batch、`runtime_info`、stage handoff | MCPMO-3a/3b | `stage_input_processors/minicpmo_4_5_omni.py` → `minicpmo_4_5_omni.py::MiniCPMO45OmniForConditionalGeneration` → TTS/code2wav | | Talker codec sampling、repetition penalty、request RNG/compaction | MCPMO-3c | `minicpmo_4_5_omni_tts.py::{make_omni_output,_sample_audio_codes,_apply_batched_repetition_penalty}` → `test_talker_batching.py` | | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | -| instructions/persona/voice/mode update、prefill slots、context lock | MCPMO-4b | `experimental/fullduplex/minicpmo45/session.py` → runtime adapter/session runner | +| instructions/native-mode update、first-append reservation、context lock | [`MCPMO-4b`](rules-duplex.md#mcpmo-4b-native-duplex-session-update-必须如实拒绝不可重建的上下文) | `experimental/fullduplex/minicpmo45/{adapter,session}.py` → runtime adapter/session runner | | duplex Talker chunk、runtime `min_tokens`、chat sampling isolation | [`MCPMO-4c`](rules-duplex.md#mcpmo-4c-native-duplex-talker-必须按-generate_chunk-预算终止) | `experimental/fullduplex/minicpmo45/adapter.py` → stage 1 sampling → Talker `generate_chunk` | | shipping YAML、duplex/chat 共服、session capacity、async scheduler | [`MCPMO-4e`](rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → config merge → realtime/chat entrypoints | | Daily-Omni/Seed-TTS accuracy、simplex/duplex perf、TTFT/TTFP/RTF | MCPMO-5a | `tests/e2e/accuracy/minicpmo_4_5/`、`tests/dfx/perf/tests/test_minicpmo_4_5*.json` → `run_benchmark.py` | @@ -241,16 +241,6 @@ confidence: high 每轮 SPEAK，应重放已验证的完整 active-speech window；任意较短 mid-utterance slice 合法返回 LISTEN，不能把它当模型失败或修改生产决策阈值。 ^[PR #6056] ^[PR #6154] ^[PR #6170] -## MCPMO-4b — session update 与派生 reservation 原子提交 - -- 触发：运行中更新 instructions、persona、voice、native mode 或影响 prefill slot 的字段。 -- 强制：先重算并验证 `prefill_slots`/first-append context tokens 等派生状态，再 ACK 并原子提交； - `native_context_locked` 只在真实 append 成功后设置。route-defining native mode 在 session 内不可变。 -- 禁止：buffered/deferred/rejected append 提前锁定；配置已变而 reservation 保持旧值；用 - true→false 绕过 native context lock。 -- 验收：成功更新同时改变派生状态；资源不足时无部分 mutation；buffer/defer/reject 均不锁， - append 成功才锁，mode flip 明确拒绝。 ^[PR #6318] - ## MCPMO-4d — async chunk 快照替换必须清理旧音频 - 触发：MiniCPM-o 4.5 native duplex 遇到重复 `turn_end` 或新的 Code2Wav async segment。 diff --git a/knowledge/repos/vllm-omni/models/personaplex/rules.md b/knowledge/repos/vllm-omni/models/personaplex/rules.md index 6e27fa2c0..7e850e56a 100644 --- a/knowledge/repos/vllm-omni/models/personaplex/rules.md +++ b/knowledge/repos/vllm-omni/models/personaplex/rules.md @@ -1,10 +1,10 @@ --- title: "PersonaPlex 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #4771", vllm_omni/experimental/fullduplex/personaplex/DESIGN.md, vllm_omni/model_executor/stage_input_processors/personaplex.py, tests/e2e/features/fullduplex/] +sources: ["PR #4771", "PR #6318", vllm_omni/experimental/fullduplex/personaplex/DESIGN.md, vllm_omni/experimental/fullduplex/personaplex/serving_adapter.py, vllm_omni/model_executor/stage_input_processors/personaplex.py, tests/e2e/features/fullduplex/, tests/entrypoints/openai_api/test_duplex_handler.py] confidence: high --- @@ -35,6 +35,17 @@ confidence: high - 验收：两 session 交错产生独立非空输出，第三个在上限处拒绝；关闭一条不改变另一条， replacement 用不同 persona 且无 state leak；异常、timeout、abort 路径无遗留。^[PR #4771] +## PPLEX-1c — persona 与 voice 在 session 创建后不可变 + +- 触发：PersonaPlex `session.update` 修改 `instructions` 或 `voice`。 +- 强制：runtime adapter 对 changed persona/voice 分别返回 `persona_update_unsupported`/ + `voice_update_unsupported`；相同值可保留。PersonaPlex 还必须拒绝 MiniCPM-o 专属的 + `minicpmo45_native_duplex` client flag，不能让它改变 append path 或绕过 adapter。 +- 禁止：ACK 变更却沿用旧 `personaplex_prefill_slots`；让 MiniCPM-o 的 `ref_audio` 检查抢先产生 + 错误码；在未原子重建 persona/voice 派生 reservation 前开放更新。 +- 验收：覆盖不同长度 persona、changed/unchanged voice 和 MiniCPM-o opt-out flag，断言命中 + PersonaPlex 专属 typed error，且并发 session、容量、reuse 与音频输出不回归。^[PR #6318] + ## PPLEX-2a — standalone 单会话 lease 必须覆盖不可取消的线程调用 - 触发：standalone `DuplexServer`（`batch_size=1`）的 `/api/chat` 或 From 2298e0145d353086a556642ee212ebd7bdf5f8ff Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:40:15 +0800 Subject: [PATCH 323/465] knowledge(vllm-omni): sync 2d4724eb (PR #5866) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 4 ++-- .../components/diffusion/rules-attention.md | 12 +++++++++++- .../repos/vllm-omni/components/diffusion/rules.md | 4 ++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 24 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 96f63f516..ea6060046 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 926e7fbfa2340d033016be79bc5b57dff9152a20 + previous_audited_sha: 91dd7fcaec7419cfc93586f2cc2dca1b09240a4e audited_ref: main - audited_sha: 91dd7fcaec7419cfc93586f2cc2dca1b09240a4e + audited_sha: 2d4724eb74b26e5a0420d87be2da9c9ba73804db # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ee5eafbf8..8ea8cd4bf 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -91dd7fcaec7419cfc93586f2cc2dca1b09240a4e`. The canonical machine baseline +2d4724eb74b26e5a0420d87be2da9c9ba73804db`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fb11770fa..2f816c25e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 91dd7fca` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2d4724eb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9274331ec..17f065319 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 91dd7fca` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 2d4724eb` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -37,7 +37,7 @@ sources: [] | Wan VAE height/width spatial reshard、empty tail、attention extent | [Wan spatial-shard rules](rules-wan-spatial-shard.md) | | multi-DiT、dotted `_dit_modules`、loader-to-offloader handoff 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | -| local FlashAttention deterministic opt-in、NPU packed mask-free/laser fallback、TRTLLM packed-padding 与 config propagation | [attention rules](rules-attention.md) | +| local FlashAttention deterministic opt-in、role-aware masked cross-attention、NPU packed mask-free/laser fallback、TRTLLM packed-padding 与 config propagation | [attention rules](rules-attention.md) | | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | | video/audio mux、DLO DP wave、result queue、async pump、SHM ownership、async-output wait 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index cdba44e96..4b3744e72 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] +sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] confidence: high --- @@ -105,6 +105,16 @@ confidence: high - 禁止：因为 cross-attention 的 Q/K 长度不同就丢弃 mask；把只匹配 query 长度的二维 mask 交给 Flash varlen；把 backend 的 `supports_attention_mask` 能力等同于支持任意 mask layout。 - 验收：CPU/mock 覆盖 masked self-attention、masked unequal-length cross-attention 与 unmasked path，分别断言 SDPA/native dispatch 和 mask shape；真实 SP 再以带 padding 的 LTX 输出对照未分片参考。^[PR #5500] +## DIFF-4v — FLASH_ATTN masked varlen packing 必须由 attention role 选择 + +- 触发：修改 diffusion `FLASH_ATTN` 的 masked-varlen unpad/pad 路径、`Attention` role 传递，或给 cross-attention 增加二维 key-padding mask。 +- 强制：只有 self-attention 才可用同一 2D mask unpad Q/K/V；构造参数精确为 `role="cross"` + 时 mask 只 unpad K/V，Q 保持 dense，并按每个 batch 的完整 query length 构造 + `cu_seqlens_q`，输出直接 reshape 回 `[B, Q, ...]`。该提交没有扩展 namespaced + `role_category="cross"` caller；LTX strict-SP 的 SDPA fallback 仍由 [DIFF-1k](#diff-1k-非等长-sp-attention-必须保留-kv-mask-并走-mask-safe-backend) 约束。 +- 禁止：以 `Q == K` 推断 self-attention，或把 key-mask indices 用于 query gather/scatter；这会丢弃 query rows，且在 batch 大于一时可能令 query 对齐到另一样本的 K/V。 +- 验收：CPU/mock 覆盖 self 与 `role="cross"` 的 masked varlen routing（包括 Q/K 等长），精确断言 cross 的 dense Q offsets 与 K/V offsets；GPU 将不同 batch valid-K lengths 的 cross output 与 SDPA 对照。^[PR #5866] + ## DIFF-1n — cuDNN attention 的单 token 形状必须提前走 MATH - 触发：修改 cuDNN diffusion attention dispatch，或引入可能出现单 token 的动态 Q/K 序列长度。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 9afbc9643..fdb5a611f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -32,7 +32,7 @@ confidence: high | loader-owned offload plan、backend enable/prefetch failure、fresh canonical retry | `checkpoint-distributed`：`DIFF-12a` | `offloader/startup.py` → `offloader/__init__.py::enable_offload_backend` → runner | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | Wan VAE spatial shard、gather/trim/reshard、empty tail、attention extent | `distributed-vae`：`DIFF-3b` | `distributed/autoencoders/wan_spatial_shard.py` → patched decoder attention/conv → gather final frame | -| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、预算与 admission | `system-runtime`：`DIFF-4a`–`4j` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | +| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、masked cross-attention、预算与 admission | `system-runtime`：`DIFF-4a`–`4j`, `DIFF-4v` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | | image/diffusion service time、forward time、missing-versus-zero、per-step metric | [`metrics-evidence`](rules-metrics.md)：`DIFF-5a`, `DIFF-5b` | output formatter/sampling metadata → stage stats aggregation → serving Prometheus emission | @@ -43,7 +43,7 @@ confidence: high | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s`, `2y`, `2z` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r`, `2w`, `12a` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o`, `2x` 见 [LoRA](rules-lora.md) | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j`（`4a`–`4i` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)） | +| `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j`、`4v`（`4a`–`4i` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)，`4v` 见 [attention rules](rules-attention.md)） | | [`metrics-evidence`](rules-metrics.md) | diffusion timing、step normalization、missing-versus-zero、benchmark interpretation | `DIFF-5a`, `DIFF-5b` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index af0421588..46f7949d4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 91dd7fca` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2d4724eb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7ab665c1e..cc2d5ebff 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 91dd7fca` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2d4724eb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9cb48c74c..e227f0a39 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 91dd7fca` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2d4724eb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9857e1842..024f0a90d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 91dd7fca` 验证存在 +- 源码校验：以上路径均已在 `main @ 2d4724eb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 796d0a06b..d5e51b54f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 91dd7fca`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2d4724eb`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 403eeceb4ad490ecbef5eacc2aee8f7d1ea06f64 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:48:08 +0800 Subject: [PATCH 324/465] knowledge(vllm-omni): sync c2723979 (PR #5742) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 3 +- .../components/serving/rules-stage-startup.md | 35 +++++++++++++++++++ .../vllm-omni/components/serving/rules.md | 14 -------- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 11 files changed, 46 insertions(+), 24 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ea6060046..1155be940 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 91dd7fcaec7419cfc93586f2cc2dca1b09240a4e + previous_audited_sha: 2d4724eb74b26e5a0420d87be2da9c9ba73804db audited_ref: main - audited_sha: 2d4724eb74b26e5a0420d87be2da9c9ba73804db + audited_sha: c2723979c0a559b71ca719366c83c304b68d5116 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8ea8cd4bf..0a1be7e80 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2d4724eb74b26e5a0420d87be2da9c9ba73804db`. The canonical machine baseline +c2723979c0a559b71ca719366c83c304b68d5116`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2f816c25e..46ebb9624 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2d4724eb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c2723979` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 17f065319..cee253cfb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2d4724eb` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c2723979` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 46f7949d4..c4cb005d1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2d4724eb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c2723979` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cc2d5ebff..c3e44fd6d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2d4724eb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c2723979` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e227f0a39..3558ca7a1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2d4724eb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c2723979` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 024f0a90d..9337a6014 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2d4724eb` 验证存在 +- 源码校验：以上路径均已在 `main @ c2723979` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -33,4 +33,5 @@ sources: [] | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | | engine startup/shutdown、stage lifecycle、full-duplex、CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | +| stage config→EngineArgs projection、explicit `devices`、TP/local-DP/PP、replica layout 或 worker 创建前的 layout guard | [stage 启动与设备布局规则](rules-stage-startup.md) | | 公开请求字段的校验、来源冲突、alias/extras 归一与 consumer view | [请求输入合同](rules-request-input.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md b/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md new file mode 100644 index 000000000..2e5eb0086 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md @@ -0,0 +1,35 @@ +--- +title: "Stage 启动与设备布局规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #6050", "PR #5445", "Issue #5003", "PR #5742", vllm_omni/engine/stage_init_utils.py, tests/engine/test_stage_device_layout.py] +confidence: high +--- + +# Stage 启动与设备布局规则 + +本页补充 [Serving 规则](rules.md) 的 stage 启动边界；stage 生命周期和子进程环境 scope +见 [engine 生命周期规则](rules-engine-lifecycle.md)。 + +## SERV-10a — stage config 到 EngineArgs 的投影必须显式且由 owner 进程物化 + +- 触发：修改 `stage_init_utils.py` 的 typed stage→`EngineArgs` projection、upstream config 字段映射或 stage startup 的终态配置构造。 +- 强制：AR/generation 从 upstream config dataclass 与 `EngineArgs` 的交集生成 projection map，显式记录且只投影构造时提供的字段，并集中处理 `cache_dtype`→`kv_cache_dtype`、`policy`→`scheduling_policy`、`data_parallel_master_ip`→`data_parallel_address` 等 alias；`CompilationConfig`/`ProfilerConfig` 以具体 upstream object 复制传递，最终 `VllmConfig` 由 engine-owning process 物化，diffusion 保持既有固定 owner 与 projection surface。 +- 禁止：维护会随 upstream schema 漂移的静态 allowlist；把 inherited defaults、runtime-owned `worker_cls`/`distributed_executor_backend`、topology-owned `scheduler_cls` 或 private API-process 字段静默送入 EngineArgs；显式提供但没有合法 projection 的字段不得被丢弃，也不得在 head process 解析设备、rank、端口或 backend。 +- 验收：非默认 upstream 字段能在最终 EngineArgs 读回且 alias 正确，未显式提供的 inherited defaults 不出现；显式无 projection 字段抛出明确错误；typed projection 保留 compilation/profiler 的 concrete type，structured stage registration 可 msgpack 传输，并回归 legacy/structured effective parity。 ^[PR #6050] + +## SERV-12a — 多副本 stage 省略 devices 时仍按 replica 数生成设备槽位 + +- 触发：修改 `split_devices_for_replicas()`、`compute_replica_layout()`、`StageRuntime._build_logical_stage_init_plans()`，或修改 stage 的 `num_replicas` 与可选 `devices` 语义。 +- 强制：当 stage 未声明 `devices` 时，返回与有效 replica 数对应的 `None` 槽位（至少保留一个槽位），使 `replica_devices_map` 和启动计划可按每个 `replica_id` 索引；每个 replica 继承 launcher 的 `CUDA_VISIBLE_DEVICES`，显式设备配置继续按既有规则拆分。 +- 禁止：因 `devices=None` 只返回单元素列表；让多副本 stage 的启动计划索引越界；将未声明设备误当成显式设备，或改变显式设备配置与既有 `ValueError` 语义。 +- 验收：覆盖 1、2、4 副本的未声明设备拆分、三副本 `replica_devices_map` 以及 logical stage init plans 的全部 replica ID 和 `None` 设备值；回归显式设备拆分、模板/池模式及错误路径。^[PR #5445] + +## SERV-12b — 显式 stage devices 必须在 worker 创建前匹配本地副本世界大小 + +- 触发：修改 `build_vllm_config()`、`_check_stage_device_layout()`、`get_stage_devices_per_replica()`、`split_devices_for_replicas()`、`compute_replica_layout()`，或修改 stage 的 `devices`、TP、DP、PP、`num_replicas` 解析与启动顺序。 +- 强制：对声明 `runtime.devices` 的每个 LLM stage，在 `create_engine_config` 或 worker/executor 创建前，以同一公式验证并拆分设备：每个本地 replica 需要 `tensor_parallel_size × data_parallel_size_local × pipeline_parallel_size`；`data_parallel_size_local` 未设置时才回退到全局 DP，值为 `0` 的 head process 不做本地设备验证。显式列表可为单 replica template（恰为每副本大小）或完整 pool（该大小 × `num_replicas`）；未声明 `devices` 时保留 launcher 分配语义。仅当去掉 TP 后设备数恰好有效，错误才说明 top-level TP 会广播到全部 stage，并要求在每个 stage override 中一并设置 TP 与 `devices`；其他 TP/DP/PP/replica 不匹配给通用维度说明。 +- 禁止：按集群全局 DP 校验本地 `runtime.devices`；让 guard 与 replica splitter 使用不同的每副本宽度；在 engine config/worker 启动后才暴露 `local rank ... out of bounds`；把 PP 或其他布局错误一律归因于 top-level TP；因为没有显式 devices 而拒绝 vLLM 的设备分配。 +- 验收：覆盖 TP 广播的单卡 stage 在 engine-config/executor 前失败、匹配 TP/device 通过、未声明 devices 跳过、local-DP 小于 global-DP 与 local-DP=0、PP/local-DP 加多副本的 template 和 full-pool 拆分，以及 PP-only mismatch 不含 TP 专属 workaround。^[Issue #5003] ^[PR #5742] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 7dc5c40bb..fddca0fc5 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -237,23 +237,9 @@ confidence: high - 验收：同长度覆盖、并发改写、named+inline、speaker cache 和 prefix hit 均不复用旧内容； 非法路径不 stat，日志只含截断值且命中真实 logger hierarchy。 ^[PR #5670] -### SERV-10a — stage config 到 EngineArgs 的投影必须显式且由 owner 进程物化 - -- 触发：修改 `stage_init_utils.py` 的 typed stage→`EngineArgs` projection、upstream config 字段映射或 stage startup 的终态配置构造。 -- 强制：AR/generation 从 upstream config dataclass 与 `EngineArgs` 的交集生成 projection map，显式记录且只投影构造时提供的字段，并集中处理 `cache_dtype`→`kv_cache_dtype`、`policy`→`scheduling_policy`、`data_parallel_master_ip`→`data_parallel_address` 等 alias；`CompilationConfig`/`ProfilerConfig` 以具体 upstream object 复制传递，最终 `VllmConfig` 由 engine-owning process 物化，diffusion 保持既有固定 owner 与 projection surface。 -- 禁止：维护会随 upstream schema 漂移的静态 allowlist；把 inherited defaults、runtime-owned `worker_cls`/`distributed_executor_backend`、topology-owned `scheduler_cls` 或 private API-process 字段静默送入 EngineArgs；显式提供但没有合法 projection 的字段不得被丢弃，也不得在 head process 解析设备、rank、端口或 backend。 -- 验收：非默认 upstream 字段能在最终 EngineArgs 读回且 alias 正确，未显式提供的 inherited defaults 不出现；显式无 projection 字段抛出明确错误；typed projection 保留 compilation/profiler 的 concrete type，structured stage registration 可 msgpack 传输，并回归 legacy/structured effective parity。 ^[PR #6050] - ### SERV-11a — 用户输入日志必须默认降级并受限输出 - 触发：修改 TTS、audio generation 或 diffusion chat serving 请求日志，以及 `request_logger` 或 `max_log_len` 的行为。 - 强制：INFO 级别只记录 request ID、模型类型、voice clone、reference image 和参数等元数据；用户输入仅在 `self.request_logger` 存在时通过 `logger.debug` 输出，并按 `max_log_len` 预留日志前缀长度进行截断；未配置上限时使用 200 字符默认上限。 - 禁止：在 INFO 日志中写入原始 `text` 或 `prompt`；无条件输出 DEBUG 用户内容；使用固定预览长度替代请求日志开关；在未配置 `max_log_len` 时输出无界用户输入。 - 验收：覆盖 TTS、diffusion TTS、audio generation 和 diffusion chat，断言 INFO 不含用户内容；启用 `--enable-log-requests` 时 DEBUG 才输出内容，且配置与未配置 `max_log_len` 时均遵守截断上限。 ^[PR #6329] - -### SERV-12a — 多副本 stage 省略 devices 时仍按 replica 数生成设备槽位 - -- 触发：修改 `split_devices_for_replicas()`、`compute_replica_layout()`、`StageRuntime._build_logical_stage_init_plans()`，或修改 stage 的 `num_replicas` 与可选 `devices` 语义。 -- 强制：当 stage 未声明 `devices` 时，返回与有效 replica 数对应的 `None` 槽位（至少保留一个槽位），使 `replica_devices_map` 和启动计划可按每个 `replica_id` 索引；每个 replica 继承 launcher 的 `CUDA_VISIBLE_DEVICES`，显式设备配置继续按既有规则拆分。 -- 禁止：因 `devices=None` 只返回单元素列表；让多副本 stage 的启动计划索引越界；将未声明设备误当成显式设备，或改变显式设备配置与既有 `ValueError` 语义。 -- 验收：覆盖 1、2、4 副本的未声明设备拆分、三副本 `replica_devices_map` 以及 logical stage init plans 的全部 replica ID 和 `None` 设备值；回归显式设备拆分、模板/池模式及错误路径。^[PR #5445] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d5e51b54f..5356c8fd1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2d4724eb`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c2723979`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 13d6904228cd1f6cd0d5cbf6b10f9b08c81fa69e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 21:57:59 +0800 Subject: [PATCH 325/465] knowledge(vllm-omni): sync 432b2c6c (PR #6089) --- adapters/vllm_omni/release_baseline.yaml | 8 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../vllm-omni/components/distributed/rules.md | 9 +++- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 8 ++-- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 9 +++- .../models/nemotron-voicechat/_index.md | 8 ++-- .../models/nemotron-voicechat/rules.md | 41 +++++++++++++++++++ 13 files changed, 76 insertions(+), 21 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1155be940..07c1f1673 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2d4724eb74b26e5a0420d87be2da9c9ba73804db + previous_audited_sha: c2723979c0a559b71ca719366c83c304b68d5116 audited_ref: main - audited_sha: c2723979c0a559b71ca719366c83c304b68d5116 + audited_sha: 432b2c6ca3a0bc29c9cf65b6de11196d13c2ddf7 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -19,8 +19,8 @@ inventories: count: 55 sha256: 9d5033c9a7ffd7ff636e32db0f9be2a549b7c16b034e13817e280f19a9e16769 deploy_yamls: - count: 84 - sha256: 2d8a2dcff97801aa87d08b4fbbdd6426bbe4e1c54fc767673b23d8b5ca4cb4e1 + count: 85 + sha256: 205c0e5c20888852e8abb4b83d8fa81cc59066c7010f3db374f889f879ab6160 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0a1be7e80..7619e1aaa 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c2723979c0a559b71ca719366c83c304b68d5116`. The canonical machine baseline +432b2c6ca3a0bc29c9cf65b6de11196d13c2ddf7`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 46ebb9624..c8d46c2bc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c2723979` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 432b2c6c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index cee253cfb..ab65a9966 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c2723979` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 432b2c6c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c4cb005d1..2ea828934 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c2723979` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 432b2c6c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index d83dcd486..ef64ca661 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406"] +sources: ["PR #5744", "PR #5976", "PR #6089", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406"] confidence: high --- @@ -77,6 +77,13 @@ confidence: high - 禁止：给无 hook 的直通路径增加 request snapshot/copy；让 queued task 读取后来 mutation 的 token、metadata 或 output history；在后台 terminal cleanup 无条件删除可能已属于下一 segment 的 watermark。 - 验收：old/new segment 在 background flush 前交错时，新 watermark 保持；连续入队后各 processor 观察各自 admission-time request snapshot；无 custom processor 时 task 仍引用原 request。^[PR #6021] +## DIST-1h — abort 不能让旧 sender generation 复活 + +- 触发：async-chunk sender 在 background save/connector put 期间 abort，且 external request ID 可复用。 +- 强制：queued task 与其 sender generation token 绑定；abort 取消 token，旧 task 在 put 前后均须拒绝更新 state，in-flight task 在 finally 中回收。sender-state lock 只保护 token/state transition，不能包住 payload construction 或 connector I/O。 +- 禁止：`cleanup_sender` 清空 map 后让 pending save 用默认 chunk counter 写回旧 chunk 0；让 scheduler thread 等待 connector I/O；把同一 external ID 的新 generation 误当作旧 task 的 cleanup target。 +- 验收：覆盖 pending/in-flight abort、stale put completion、ID reuse 和普通 async-chunk transport。^[PR #6089] + ## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 - 触发：上游 MoE factory/oracle 或 `init_model_parallel_group` 增加 all-to-all manager 要求。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c3e44fd6d..72024ed75 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c2723979` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 432b2c6c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3558ca7a1..ae2dd19fe 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c2723979` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 432b2c6c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 346ab5718..da8483d7a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619"] --- # Scheduler 规则 @@ -257,9 +257,9 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 ## SCHED-5g — resumable async-chunk 终态清理必须以 live queue 所有权为准 - 触发：resumable async-chunk 请求在一个流式 segment 结束时进入 `FINISHED_STOPPED`，但仍由 `running`、`waiting`、`skipped_waiting` 或 connector 隐藏队列持有，随后 session close、取消或其他路径调用 `finish_requests`。 -- 强制：先物化可能被多层消费的 request-id iterator，并在 adapter 清理前快照 `skipped_waiting` 中承担流式等待计数的请求；只对仍有活队列所有权的目标 resumable 终态请求恢复状态，`skipped_waiting` 恢复为 `WAITING_FOR_STREAMING_REQ`，`running`/`waiting` 按实际队列对齐；running purge 必须限定本次 finish 集合，并确保 `_free_request`、coordinator 与 connector 清理恰好执行一次。 -- 禁止：把任意已完成请求重新打开；对脱离所有 live queue、可能等待 deferred block free 的终态请求调用释放；全局清空 running 中无关的 resumable segment；重复消费单遍 request-id iterator，或因非流式 skipped 请求错误减少 streaming counter。 -- 验收：AR 与 generation scheduler 均覆盖 hidden、`running`、`waiting`、`skipped_waiting`、脱离队列及无关 resumable 请求；断言状态、队列、计数、request map 和 active-stream capacity 正确，首次 finish 恰好释放、第二次无操作，并验证 generator request-id 输入。^[PR #6360] +- 强制：先物化可能被多层消费的 request-id iterator，并在 adapter 清理前快照 `skipped_waiting` 中承担流式等待计数的请求；只对仍有活队列所有权的目标 resumable 终态请求恢复状态，`skipped_waiting` 恢复为 `WAITING_FOR_STREAMING_REQ`，`running`/`waiting` 按实际队列对齐；下游 async-chunk 的 segment stop 必须先清除该 segment 的 finished 标记，再从 `skipped_waiting` 转为 ordinary `WAITING` 并恢复 connector polling，同一 update 尾部按 stale stopped 集合清理时不得移除这个已重新入队的 request；running purge 必须限定本次 finish 集合，并确保 `_free_request`、coordinator 与 connector 清理恰好执行一次。 +- 禁止：把任意已完成请求重新打开；对脱离所有 live queue、可能等待 deferred block free 的终态请求调用释放；因 request 在本轮进入时是 `WAITING_FOR_CHUNK` 就撤销其同轮 requeue；全局清空 running 中无关的 resumable segment；重复消费单遍 request-id iterator，或因非流式 skipped 请求错误减少 streaming counter。 +- 验收：AR 与 generation scheduler 均覆盖 hidden、`running`、`waiting`、`skipped_waiting`、脱离队列及无关 resumable 请求；另覆盖下游 receiver 从 segment stop 同轮转回 `WAITING`，断言它保留在 waiting queue、离开 skipped queue、流式等待计数递减且 segment-finished 标记清除；首次 finish 恰好释放、第二次无操作，并验证 generator request-id 输入。^[PR #6089] ^[PR #6360] ## SCHED-6a — async discard 的计数单位必须与 stale drain 一致 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9337a6014..5792f61bb 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c2723979` 验证存在 +- 源码校验：以上路径均已在 `main @ 432b2c6c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5356c8fd1..11646c263 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c2723979`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 432b2c6c`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -38,7 +38,12 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | | Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 40 个模型族目录 | | Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 55 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 84 份 | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 85 份 | + +PR #6089 新增 `nemotron_labs_voicechat_duplex.yaml`，因此只增加 deploy profile；它没有 +增加 AR/diffusion registry architecture 或 `OMNI_PIPELINES` key。该 profile 是原生全双工 +Nemotron VoiceChat 的单会话 eager fp32 配置，不能从 YAML 或 PR 的作者测试外推 wall-clock +realtime、确定性 barge-in 或一般硬件支持。^[PR #6089] 对比上一审计快照（`39c16d75`）：AR 架构 88→81，diffusion pipeline 65→64， OMNI_PIPELINES 57→55，deploy 86→84。PR #6353 删除 AudioX 的 diffusion registry、 diff --git a/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md b/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md index f655b09b0..54c795fae 100644 --- a/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md +++ b/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: index tags: [vllm-omni, models, serving] -sources: [vllm_omni/deploy/nemotron_labs_voicechat.yaml, vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/asr/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py] +sources: ["PR #6089", vllm_omni/deploy/nemotron_labs_voicechat.yaml, vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/asr/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py] confidence: high --- @@ -31,7 +31,9 @@ confidence: high ## 依赖的共享代码模块 -- `vllm_omni/experimental/fullduplex/` → 无对应 component owner +- `vllm_omni/experimental/fullduplex/openai/` → [serving](../../components/serving/_index.md) +- `vllm_omni/core/sched/` → [scheduler](../../components/scheduler/_index.md) +- `vllm_omni/distributed/omni_connectors/` → [distributed](../../components/distributed/_index.md) - `vllm_omni/model_executor/models/` → [model-executor](../../components/model-executor/_index.md) - `vllm_omni/config/stage_config/` → [configuration](../../components/configuration/_index.md) @@ -53,4 +55,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | +| 原生 duplex、帧/工具合同与验证边界 | [Nemotron VoiceChat rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md b/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md new file mode 100644 index 000000000..b2ef220c2 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md @@ -0,0 +1,41 @@ +--- +title: "Nemotron VoiceChat 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, serving] +sources: ["PR #6089", vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/experimental/fullduplex/openai/, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py, tests/e2e/features/fullduplex/nemotron_voicechat/, tests/e2e/online_serving/test_nemotron_voicechat_duplex.py] +confidence: high +--- + +# Nemotron VoiceChat 规则 + +只有 `NVC-数字字母` 是本页可审计规则 ID。 + +## NVC-1a — 原生 duplex 是 frame-locked、可恢复的三 stage 数据面 + +- 触发：修改 duplex deploy、native runtime extension、Stage-0 append，或 Thinker→Talker→Code2Wav handoff。 +- 强制：只接受每 80 ms 一帧的 1280-sample/16 kHz PCM 输入；Stage 0 跨 wake 保留可恢复 request，text/function side channel 只消费 scheduler 在本次 wake 冻结的 `new_token_ids`。Talker 的累计 code history 在送入有状态 Code2Wav 前只传未见 suffix；native `codec_streaming` 可在当前累计 timeline 耗尽时结束一个 segment，offline async-stream 则必须同时等 `upstream_finished`，包括零进度 wake。共享的下游 receiver requeue/cleanup 遵循 [SCHED-5g](../../components/scheduler/rules.md#sched-5g-resumable-async-chunk-终态清理必须以-live-queue-所有权为准)。 +- 禁止：把 scheduler segment finish 当作 codec stream lifetime finish；重放累计 code prefix 到 causal codec cache；从 `supports_core_resumable_request` 推出公开 reusable KV lease（该 adapter 明确 `supports_core_kv_lease=false`）。 +- 验收：固定 PCM contract、append/runtime contract、scheduler resume 和 data-plane tests 均须通过；offline async-stream completion 也须保持既有终端语义。^[PR #6089] + +## NVC-1b — native 工具调用由客户端执行，失败投递必须可重试 + +- 触发：修改 `AVAILABLE_TOOLS`/`TOOLCALL`/`TOOL_RESPONSE` prompt、Realtime `function_call_output` 或 runtime update。 +- 强制：服务器只验证、排队并 frame-locked reinject 客户端拥有的 function result，绝不执行任意工具；允许多个到达重叠的 result 形成有序 batch。仅在 runtime update 成功且已发出 `conversation.item.created` 后记录 call output，以便投递失败后同一 `call_id` 可重试。 +- 禁止：第二个 tool result 抛出 stage-fatal 异常；在 delivery 前写 duplicate-output ledger；把 client tool result 解释为 server-side execution authorization。 +- 验收：覆盖多个 response batch、duplicate/invalid output、delivery failure 后 retry 与 realtime event ordering。^[PR #6089] + +## NVC-1c — codec suffix 状态必须随 request lifetime 回收 + +- 触发：修改 `talker2code2wav_async_chunk` 的 cumulative-history slicing、causal codec cache，或 native session abort。 +- 强制：按 external request ID 记录已见 frame 数和有状态 decoder cache，只把累计 code history 的未见 suffix 送入 decoder；abort/terminal cleanup 必须同时清除 processor payload、chunk watermark 和 codec state，使复用同一 external ID 的新 request 从首帧开始。共享 sender-generation fence 遵循 [DIST-1h](../../components/distributed/rules.md#dist-1h-abort-不能让旧-sender-generation-复活)。 +- 禁止：每次 wake 重放累计 prefix；abort 后继承旧 `frames_seen`/cache；把普通 scheduler segment finish 当成 native codec lifetime finish。 +- 验收：覆盖累计输入只解码 suffix、segment 边界保留 cache、abort 清空后同 ID 首帧不混入旧 codes，以及 pending/in-flight sender race 的共享回归。^[PR #6089] + +## NVC-1d — full-model 音频证据和部署声明必须限界 + +- 触发：修改 full-model E2E gate、Buildkite selection 或 duplex profile。 +- 强制：full-model probe 除 response/event 完成外，至少断言 PCM RMS floor、最小 audio packet 数和与 input frame 数相关的 duration/bytes；offline streaming 要比较其原有完整 audio contract。profile 的 `max_num_seqs: 1` 只证明默认 admission 配置。 +- 禁止：用非空 bytes、约 1 LSB 的 RMS 或文本成功证明音频可用；把作者在 H200 实测、generic nightly selector 或 YAML 存在写成 Buildkite pass、wall-clock realtime、确定性 barge-in 或跨硬件保证。 +- 验收：PR 作者报告 targeted H200 suite 306 passed，reviewer 后续报告 399 CPU tests、offline WAV byte-identical 和 native duplex non-silent audio；这些是提交时证据，当前 KB validation 不能复现真实 checkpoint/H200 E2E。Buildkite 改动只把既有 AMD-ready 步骤的超时从 20 分钟增至 45 分钟，不新增该模型专属 pipeline。^[PR #6089] From 51e704a6535ab6dd16be5f7b909bf50271df3c8a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:04:19 +0800 Subject: [PATCH 326/465] knowledge(vllm-omni): sync 0e44fc74 (PR #6162) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 10 ++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-checkpoint-loading.md | 28 ++++++++++++++++++- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../docs/generation-and-support-rules.md | 9 +++++- knowledge/repos/vllm-omni/models/catalog.md | 2 +- 12 files changed, 54 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 07c1f1673..33e1c8020 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c2723979c0a559b71ca719366c83c304b68d5116 + previous_audited_sha: 432b2c6ca3a0bc29c9cf65b6de11196d13c2ddf7 audited_ref: main - audited_sha: 432b2c6ca3a0bc29c9cf65b6de11196d13c2ddf7 + audited_sha: 0e44fc7406c8cdaab68065fa35c3533470392d13 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7619e1aaa..e59673531 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -432b2c6ca3a0bc29c9cf65b6de11196d13c2ddf7`. The canonical machine baseline +0e44fc7406c8cdaab68065fa35c3533470392d13`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index fb284c37b..a116f8c5a 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json] --- # vLLM-Omni CI @@ -33,6 +33,14 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie shape, and nonzero variance. It is an offline inference smoke, not online-serving, FLUX, CUDA, or general backend coverage. ^[PR #5544] +## SVDQuant W4A4 coverage boundary + +- The SVDQuant suite is CPU-marked and uses mocked NVFP4 kernels to check configuration, + tensor conversion, the BF16 correction calculation, and Column/Row/QKV TP loading. It does + not run a compatible real kernel, load a released checkpoint, or establish SM103 execution, + output quality, latency, or VRAM results. Any GPU support or performance claim requires a + separately recorded exact-device E2E run. ^[PR #6162] + ## Diffusion GGUF nightly plugin compatibility - The L4 full-model diffusion-quantization nightly lane installs diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c8d46c2bc..8b392b1ec 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 432b2c6c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0e44fc74` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ab65a9966..f73e651a9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 432b2c6c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 0e44fc74` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index cd1c9283c..721f5e7d2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #4061"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6162", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, vllm_omni/quantization/svdquant_config.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #4061"] confidence: high --- @@ -208,3 +208,29 @@ confidence: high restored tensor byte/backing-pointer equality；验证 checkpoint mmap control path 不变。另覆盖 registration capability/budget/pin-memory gates、partial registration rollback、direct/staged transfer parity、没有 staging allocation 的成功路径，以及 teardown 前 unregister。^[PR #6486] ^[PR #6591] + +## DIFF-2aa — serialized SVDQuant 只能按 NVFP4 checkpoint ABI 加载 + +- 触发：修改 `quant_method="svdquant"` 的 diffusion checkpoint 解析、NVFP4 kernel 选择、 + SVDQuant parameter 创建/加载、TP shard，或尝试扩展其 dtype、GPU 或 component 范围。 +- 强制：factory 必须从 transformer 的 serialized `quantization_config` 构造 SVDQuant config； + Phase 1 只接受 positive `rank`、`precision="nvfp4"` 和 signed BF16 activation。每个 + TP-local input width 必须能被 16 整除；加载 `qweight`、`wscales`、`proj_down`、`proj_up`、 + `smooth_factor`、`wcscales` 和 `wtscale` 时保持其 input/output shard axis。base GEMM 只能 + 复用 ABI-compatible FlashInfer、CUTLASS 或 FBGEMM NVFP4 kernel；原 activation 走 BF16 + low-rank correction，只有 base GEMM 使用 smoothed activation；可选 per-output `wcscales` + 在 base output 上逐 channel 应用后再加 correction，以保留 fused Q/K/V 的独立 outer scale。 + `get_quant_method()` 对非 `LinearBase` 必须返回 `None` 让 embedding/LM-head 等使用 upstream + fallback，只有被 skip 的 eligible linear 返回 `UnquantizedLinearMethod`。没有 serialized scale 或 + correction tensors 的 encoder/audio component 必须保持 BF16，不能从 transformer config + 隐式继承 SVDQuant。 +- 禁止：在 model load 时 calibration/conversion；接受 FP16、unsigned activation、非-NVFP4 + precision 或不兼容 forced kernel 后延迟到首个 forward 才失败；把 compatibility path 描述为 + fused GEMM+correction，或把 SM103 之外的 GPU、其他 checkpoint layout、质量/性能结果写成 + 已验证支持。 +- 验收：mock/config tests 覆盖 config rejection、skip list、canonical tensor shape/dtype、 + compatible-kernel rejection、BF16 base-plus-correction numerical path，以及 Column/Row/QKV + TP sharding；GPU 证据还必须在 exact checkpoint、backend、device、seed 和 quality workload + 上比较真实 kernel 输出。当前合入测试未执行真实 kernel 或 released checkpoint E2E，故仅证明 + loader contract；PR review 所要求的 frozen-head BF16-vs-SVDQuant quality/performance comparison + 没有随本提交提供。^[PR #6162] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2ea828934..4f0903ec7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 432b2c6c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0e44fc74` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 72024ed75..0a7a71dad 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 432b2c6c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0e44fc74` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ae2dd19fe..1bc98b68f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 432b2c6c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0e44fc74` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5792f61bb..959074505 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 432b2c6c` 验证存在 +- 源码校验：以上路径均已在 `main @ 0e44fc74` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md index 1219e0c49..c0f962e14 100644 --- a/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md +++ b/knowledge/repos/vllm-omni/docs/generation-and-support-rules.md @@ -4,7 +4,7 @@ created: 2026-08-10 updated: 2026-09-04 type: rule tags: [vllm-omni, docs] -sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, examples/online_serving/text_to_image/, examples/online_serving/image_to_video/, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998", "PR #6045", "PR #6049", "PR #6074", docs/features/README.md, docs/design/index.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, docs/user_guide/quantization/overview.md, docs/user_guide/quantization/bitsandbytes.md, docs/user_guide/quantization/gguf.md] +sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, docs/mkdocs/hooks/generate_examples.py, docs/models/supported_models.md, docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md, examples/online_serving/diffusers_pipeline_adapter/README.md, examples/online_serving/text_to_image/, examples/online_serving/image_to_video/, recipes/README.md, tests/docs/test_generate_examples.py, "PR #5969", "PR #5987", "PR #5998", "PR #6045", "PR #6049", "PR #6074", "PR #6162", docs/features/README.md, docs/design/index.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md, docs/user_guide/quantization/overview.md, docs/user_guide/quantization/bitsandbytes.md, docs/user_guide/quantization/gguf.md, docs/user_guide/quantization/svdquant.md] --- # Generated docs and supported-model evidence rules @@ -92,6 +92,13 @@ sources: [.claude/skills/quantization/references/modelopt-fp8.md, docs/.nav.yml, 是 grouped 仅 TORCH_SDPA、仅 `max_num_seqs=1`；HSDP 与 Ulysses/CFG 可组合但与 TP 不兼容；local layerwise offload 与 multi-device DLO 分表。docs-only build 不证明 runtime；overview 移除 GGUF 只同步 core 导航归属，保留的 GGUF guide、OOT plugin 说明与实现/测试证据仍须独立核查。^[PR #6045] ^[PR #6074] +- SVDQuant guide must say that it consumes an offline serialized NVFP4 checkpoint, enumerate + the checkpoint tensor contract, and distinguish its SM103-compatible-kernel loading contract + from real-kernel quality or performance validation. Adding it to the Quantization nav does + not establish a model matrix, another GPU target, or fused-kernel support. The merged commit + adds only the dedicated user guide/nav entry; the review request to update + `docs/design/feature/quantization.md` and `docs/user_guide/quantization/overview.md` remains an + unresolved thread, so those broader matrices cannot be treated as synchronized evidence. ^[PR #6162] ## DOCGEN-1d — shared task example 不得把模型默认伪装成通用合同 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 11646c263..88188d790 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 432b2c6c`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0e44fc74`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 From 62a0a0967f98ebfef14f6b9d33875adb7bf9e846 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:10:20 +0800 Subject: [PATCH 327/465] knowledge(vllm-omni): sync da4a08b6 (PR #6651) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-checkpoint-loading.md | 6 ++-- .../vllm-omni/components/diffusion/rules.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/host-weight-runtime/rules.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- .../vllm-omni/models/flux2-klein/_index.md | 13 ++++++-- .../vllm-omni/models/flux2-klein/rules.md | 33 +++++++++++++++++++ 14 files changed, 59 insertions(+), 17 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/flux2-klein/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 33e1c8020..f304eb7a6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 432b2c6ca3a0bc29c9cf65b6de11196d13c2ddf7 + previous_audited_sha: 0e44fc7406c8cdaab68065fa35c3533470392d13 audited_ref: main - audited_sha: 0e44fc7406c8cdaab68065fa35c3533470392d13 + audited_sha: da4a08b65bfe717d8c4bc1a98decd0c8ad88976b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e59673531..ff4277b9f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0e44fc7406c8cdaab68065fa35c3533470392d13`. The canonical machine baseline +da4a08b65bfe717d8c4bc1a98decd0c8ad88976b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8b392b1ec..c15397400 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0e44fc74` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ da4a08b6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f73e651a9..6e348230b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0e44fc74` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ da4a08b6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index 721f5e7d2..73f78d015 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6162", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, vllm_omni/quantization/svdquant_config.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #4061"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6162", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", "PR #6651", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, vllm_omni/quantization/svdquant_config.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #4061"] confidence: high --- @@ -207,7 +207,9 @@ confidence: high source rejection、warm path zero ordinary DiT materialization，以及 shared finalization 前后 restored tensor byte/backing-pointer equality；验证 checkpoint mmap control path 不变。另覆盖 registration capability/budget/pin-memory gates、partial registration rollback、direct/staged transfer parity、没有 - staging allocation 的成功路径，以及 teardown 前 unregister。^[PR #6486] ^[PR #6591] + staging allocation 的成功路径，以及 teardown 前 unregister。FLUX.2-klein 的 consumer-owned + validator 与范围边界见 [FLUXK-1a](../../models/flux2-klein/rules.md#fluxk-1a-final-layout-hwr-必须恢复可立即使用的-bf16-klein-transformer)；该接入复用共享 HWR/DLO 协议， + 不扩大协议本身。^[PR #6486] ^[PR #6591] ^[PR #6651] ## DIFF-2aa — serialized SVDQuant 只能按 NVFP4 checkpoint ABI 加载 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index fdb5a611f..09f24ed2c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4f0903ec7..2a94da15f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0e44fc74` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ da4a08b6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md index 2dec6f03c..63f01596e 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, components] -sources: ["PR #6419", "PR #6445", "PR #6486", "PR #6591", vllm_omni/host_weight_runtime/, "PR #6427"] +sources: ["PR #6419", "PR #6427", "PR #6445", "PR #6486", "PR #6591", vllm_omni/host_weight_runtime/] confidence: high --- diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0a7a71dad..5521fa07e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0e44fc74` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ da4a08b6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1bc98b68f..be3fc5474 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0e44fc74` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ da4a08b6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 959074505..e20a57b39 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0e44fc74` 验证存在 +- 源码校验：以上路径均已在 `main @ da4a08b6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 88188d790..337b01586 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0e44fc74`（2026-08-26）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ da4a08b6`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/models/flux2-klein/_index.md b/knowledge/repos/vllm-omni/models/flux2-klein/_index.md index 06a9fea3b..ae3119f7a 100644 --- a/knowledge/repos/vllm-omni/models/flux2-klein/_index.md +++ b/knowledge/repos/vllm-omni/models/flux2-klein/_index.md @@ -1,15 +1,15 @@ --- title: "FLUX.2-Klein（Qwen3 编码的 T2I/I2I）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/flux2_klein/, vllm_omni/diffusion/registry.py] +sources: ["PR #6651", vllm_omni/diffusion/models/flux2_klein/, vllm_omni/diffusion/registry.py] --- # FLUX.2-Klein -以下事实在 `main @ 5d44868e` 复核（源码派生页,尚无本模型的运行经验沉淀）。 +以下事实在 `main @ da4a08b6` 复核；运行证据与支持范围以具体规则为准。 ## 名称与范围 @@ -34,3 +34,10 @@ sources: [vllm_omni/diffusion/models/flux2_klein/, vllm_omni/diffusion/registry. - 审查 flux2_klein 的 Qwen3-LLM 编码链、图像输入或蒸馏旗标改动;文本编码 方案与 flux2 家族不同,勿混（见该页）。 +- 审查 final-layout Host Weight Runtime (HWR) restore、BF16 checkpoint 与 no-AllGather DLO 的组合。 + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| FLUX.2-klein final-layout HWR contract、packed QKV state、eligible/supported boundary | [Klein rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/flux2-klein/rules.md b/knowledge/repos/vllm-omni/models/flux2-klein/rules.md new file mode 100644 index 000000000..1b749d04e --- /dev/null +++ b/knowledge/repos/vllm-omni/models/flux2-klein/rules.md @@ -0,0 +1,33 @@ +--- +title: "FLUX.2-Klein 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #6651", vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py, tests/diffusion/model_loader/test_final_layout_host_weights.py] +confidence: high +--- + +# FLUX.2-Klein 规则 + +只有 `FLUXK-数字字母` 是可审计规则 ID。共享 artifact、lease 和 DLO transport +约束见 [Diffusion checkpoint rules](../../components/diffusion/rules-checkpoint-loading.md#diff-2z-final-layout-hwr-只能接入-eligible-no-allgather-dlo) 与 +[Host Weight Runtime rules](../../components/host-weight-runtime/rules.md)。 + +## FLUXK-1a — final-layout HWR 必须恢复可立即使用的 BF16 Klein transformer + +- 触发：修改 `Flux2Transformer2DModel` 的 HWR contract、constructor state、packed weight + loading、restore validation，或声称扩展 Klein 的 HWR scope。 +- 强制：使用版本化 `flux2-klein-dit` final-layout contract；constructor 在任何普通 + `load_weights()` 之前建立六项 packed Q/K/V mapping。restore validator 必须同时检查 + double-stream 与 single-stream block stacks、核心模块、CPU materialization、连续 strided + BF16 parameters，以及 persistent loader-owned `beta`/`eps` buffers；warm restore 只接管 + transformer，text encoder 与 VAE 继续 ordinary loading。 +- 禁止：依赖 `load_weights()` 重建 warm-hit 所需 mapping；接受 meta/non-CPU、非 BF16、非连续 + 参数或 non-persistent `beta`/`eps`；把 FLUX.2-klein-9B、FLUX.2-dev、online FP8、HSDP、LoRA/ + adapted weights、non-default load format 或 AllGather 写成此验证过的 contract scope。 +- 验收：reduced transformer 覆盖两类 block、six-map packed QKV、persistent buffers、publication → + exact restore 的值和 storage-pointer equality，以及每种 invalid restore invariant 的拒绝；loader + 覆盖 required miss 在 ordinary loading/publication 前失败。实际模型证据目前仅为 4B、BF16、default + format、non-HSDP、TP1/DP1/SP1 no-AllGather DLO；TP2/SP 与 9B 仍未测量，不能由同一 class 或 + identity mechanics 外推。^[PR #6651] From 7f64c445a5af0e00421a3d5f6d2a9f40b60f9e66 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:16:41 +0800 Subject: [PATCH 328/465] knowledge(vllm-omni): sync 651b0322 (PR #6606) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 4 ++++ .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/model-executor/rules.md | 1 + .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/rules.md | 10 ++++++++++ .../repos/vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/components/serving/rules.md | 1 + knowledge/repos/vllm-omni/models/catalog.md | 2 +- knowledge/repos/vllm-omni/rebase/_index.md | 7 ++++--- knowledge/repos/vllm-omni/rebase/workflow.md | 13 +++++++------ 15 files changed, 37 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f304eb7a6..ede059103 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0e44fc7406c8cdaab68065fa35c3533470392d13 + previous_audited_sha: da4a08b65bfe717d8c4bc1a98decd0c8ad88976b audited_ref: main - audited_sha: da4a08b65bfe717d8c4bc1a98decd0c8ad88976b + audited_sha: 651b03225b69fe825525e1cfa00300fe856fcea0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ff4277b9f..83b6d6542 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -da4a08b65bfe717d8c4bc1a98decd0c8ad88976b`. The canonical machine baseline +651b03225b69fe825525e1cfa00300fe856fcea0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 3b9dab06d..47d7b4007 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -179,6 +179,10 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload 同时计入测试辅助模型：主模型占卡时，Whisper 等 grader 需要 spare accelerator 才能避免 CPU 超时。 - 禁止：在 release image 上保留补偿旧 image 的依赖手术；用 Docker 默认 `/dev/shm`；只按被测模型 卡数配置 lane 而忽略 grader；把部分 nightly green 当全部兼容问题已关闭。 +- v0.28 对齐的 release-boundary example：CUDA/CI/ROCm base image 与 installed vLLM 都指向 + `v0.28.0`；CI Dockerfile 在官方同版本 image 出现后移除临时 wheel reinstall，而不是把旧 + 0.27 image 上的 ABI repair 带入 release。文档安装 pin 也必须同步；非 release SHA 才恢复 + 明确的 wheel/ABI block。^[PR #6606] - 验收：image tag 与目标 release 一致；startup 通过共享内存 preflight；目标 lane 证明实际 grader device；每个残余失败独立记录，尤其数值/质量回归不能由 API 兼容测试替代。^[PR #5976] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c15397400..5c8ba73f4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ da4a08b6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 651b0322` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6e348230b..b4ae0c78a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ da4a08b6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 651b0322` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2a94da15f..2697d0480 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ da4a08b6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 651b0322` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 5521fa07e..e58193709 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ da4a08b6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 651b0322` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 1aa6f500d..f644903b6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -25,6 +25,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | | fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | H3 text encoder fused owner；其他模型先确认目标 main 是否已有 fused parameter | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | +| upstream runner/output API drift、`num_nans`、sample-budget 或 quant loader helper moved | `async-output` + output-contract | live upstream runner/output dataclasses → `gpu_{ar,generation}_model_runner.py` / `outputs/output_processor.py` → diffusion quant configs；moved symbol 必须用目标 release 的 canonical import，而非保留旧 fallback ^[PR #6606] | | Talker-MTP FULL graph、平台 capture 能力、显式 opt-out | `mtp-graph`：`EXEC-4c` | `platforms/interface.py::supports_talker_mtp_graph_capture` → platform override → model `talker_mtp_graph_safe` → `OmniGPUModelRunner._init_talker_mtp` | | `model_extras`、shared T2I/I2V example、canonical prompt envelope | `image-task-envelope`：`EXEC-6a` | `examples/offline_inference/{text_to_image/text_to_image.py,image_to_video/image_to_video.py}` → `model_extras/registry.py::{build_text_to_image_prompt,build_image_to_video_prompt}` → model pipeline validation | | CosyVoice3 non-module TensorRT CFM、stream/event ownership、raw-pointer buffer lifetime 或 context-pool reuse | [CosyVoice3 TensorRT handoff](../../models/cosyvoice3/rules.md)：`COSYVOICE3-1a` | `code2wav_core/cfm.py::ConditionalCFM.forward_estimator` → `flow_estimator_trt.py::TrtContextWrapper` | diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index be3fc5474..abdf335d7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ da4a08b6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 651b0322` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index da8483d7a..b42aeb6ae 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -156,6 +156,16 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 Omni fast path 不重建与 generation fallback；AR/generation 分别验证 pending-input、queue restore、finished-set/abort policy。直接调用 scheduler 方法的轻量 stub 必须继承 mixin 或 显式绑定本次方法真实依赖，使断言确实到达目标分支。 ^[PR #5461] +- vLLM 0.28 的具体补充：当 `defer_block_free` 启用时，AR 与 generation scheduler 都必须在每个 + 非空 schedule step 推进 `sched_step_seq`，并在 `update_from_output()` 的开头推进 + `processed_step_seq`、调用 `_drain_deferred_frees()`；generation fast path 的推进必须早于 + `_update_after_schedule` 的 `last_sched_seq` 标记。EC connector 完成请求时先调用 + `request_finished(request)`，将 delayed-free 合入 block 生命周期，并把 + `ec_transfer_params` 经 `OmniEngineCoreOutput` 传给 output processor。prefill stats 必须在 + request 尚在 KV cache manager 中时以 `estimate_cached_tokens()` finalize，随后在所有 + `RequestOutput` 路径保留 `num_cache_creation_tokens`。这些是 upstream 0.28 合同，不是 + optional compatibility shim；同时覆盖 AR/generation 两条 override 路径和 hook-order 测试。 + ^[PR #6606] ## SCHED-4a — side-stream 复制必须拥有源 buffer 的完成期 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e20a57b39..8219eb277 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ da4a08b6` 验证存在 +- 源码校验：以上路径均已在 `main @ 651b0322` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index fddca0fc5..ba46c068c 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -26,6 +26,7 @@ confidence: high | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup acknowledgment | `engine-lifecycle`：`SERV-5a`, `SERV-5b`, `SERV-5m` | `entrypoints/async_omni.py::{AsyncOmni.pause_generation,AsyncOmni.resume_generation,AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni._abort}` → `engine/{stage_pool,orchestrator,async_omni_engine}.py` → AR EngineCore helper / diffusion worker RPC | | stage/replica death、remote membership register/watch/detach、`EngineDeadError`、`StageUnavailableError`、health/readiness、eviction | `engine-lifecycle`：[SERV-5f](rules-fault-isolation.md#serv-5f-stage-death-按-replica-隔离request-fatal-不等于-process-fatal)、[SERV-5n](rules-engine-lifecycle.md#serv-5n-remote-replica-attachdetach-必须按本次-membership-generation-判定) | `engine/membership_controller.py` register/watcher generation → `engine/stage_pool.py` live set → `engine/orchestrator.py` poll/dispatch/cleanup → `entrypoints/{omni_base,async_omni}.py` error transport | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | +| vLLM rebase、chat parser/renderer API moved or removed | `engine-lifecycle` | target-release `vllm.parser.*` / renderer API → `serving_chat.py` / `api_server.py`; remove a compatibility fallback only after the pinned base image is the target release ^[PR #6606] | | TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | | request-level LoRA、AR-only/multistage、streaming input update、stage cardinality | `request-contract`：`SERV-4l` | `serving_chat.py` LoRA resolve → `AsyncOmni.generate` → `AsyncOmniEngine._build_add_request_message` → stage-0 input processor | | caller sampling params、pipeline `sampling_constraints`、stage config runtime extraction | `request-contract`：`SERV-4o` | `OmniBase.resolve_sampling_params_list` → `_get_sampling_constraints_list` → `_apply_sampling_constraints` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 337b01586..de4f77fcd 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ da4a08b6`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 651b0322`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/rebase/_index.md b/knowledge/repos/vllm-omni/rebase/_index.md index 16e1bc866..5d357127d 100644 --- a/knowledge/repos/vllm-omni/rebase/_index.md +++ b/knowledge/repos/vllm-omni/rebase/_index.md @@ -1,16 +1,17 @@ --- title: "Rebase（对齐 upstream vLLM）" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-04 type: index tags: [vllm-omni, rebase] -sources: [".buildkite/cuda/rebase-pipeline.yml", "vllm-omni-rebase-agent@122a9468:agent/config.py"] +sources: ["PR #6606", ".buildkite/cuda/rebase-pipeline.yml", "vllm-omni-rebase-agent@122a9468:agent/config.py"] --- # Rebase（对齐 upstream vLLM） 把 vllm-omni 对齐到新版 upstream vLLM 的周期性工作：专用分支 `dev/vllm-align`、 -专用管线 `.buildkite/cuda/rebase-pipeline.yml`（`v0.26.0 @ a4ea67a2` 验证存在）、专职 +专用管线 `.buildkite/cuda/rebase-pipeline.yml`（仓库 `main @ 651b0322`、upstream vLLM +`v0.28.0 @ 2cf0a6915ce5` 验证）、专职 自动化（rebase-agent 仓库——运营系统在那里，本主题只沉淀领域知识）。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/rebase/workflow.md b/knowledge/repos/vllm-omni/rebase/workflow.md index b62ab2352..758a1c7e5 100644 --- a/knowledge/repos/vllm-omni/rebase/workflow.md +++ b/knowledge/repos/vllm-omni/rebase/workflow.md @@ -1,25 +1,26 @@ --- title: "Rebase 工作流：分支、波次与失败路由" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-04 type: guide tags: [vllm-omni, rebase] -sources: ["vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh", ".buildkite/cuda/rebase-pipeline.yml"] +sources: ["PR #6606", "vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh", ".buildkite/cuda/rebase-pipeline.yml", docker/Dockerfile.ci] --- # Rebase 工作流：分支、波次与失败路由 运营事实来自 rebase-agent 配置快照（@122a9468，**可能漂移**）；仓库侧事实在 -`v0.26.0 @ a4ea67a2` 复核。运营系统以 rebase-agent 仓库为准，本页是知识树快照 -（2026-07-16）。 +`main @ 651b0322` 复核。运营系统以 rebase-agent 仓库为准，本页是知识树快照； +upstream release baseline 为 `v0.28.0 @ 2cf0a6915ce5`。^[PR #6606] ## 分支与管线 - 对齐分支 `dev/vllm-align`（目标合回 `main`）；专用 Buildkite 管线 `.buildkite/cuda/rebase-pipeline.yml`（仓库侧）+ 运营管线 `vllm-omni-rebase` （nightly 与 main CI）、`vllm-omni-release`（CI），org `vllm`；wheel 变体 - `cu130`；上次 rebase 的 vLLM 提交 pin - `1acd67a795ebccdf9b9db7697ae9082058301657`。 + `cu130`；本轮 release-boundary rebase 的 vLLM pin 是 tag `v0.28.0` 的 + `2cf0a6915ce5`。CUDA/CI/ROCm base image 和安装文档须同步使用同一 release；只有 + 对齐未发布 SHA 时才启用额外 wheel reinstall/ABI repair。^[PR #6606] - nightly 循环：定时 nightly 跑对齐分支全量套件；注意定时 nightly 与 API build 的互杀陷阱（[ci-environment-gotchas](../ci/guides/ci-environment-gotchas.md) 第 3 条）。 From 2591f5394eed1e89ac4b35e37bc261085c2d4431 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:20:42 +0800 Subject: [PATCH 329/465] knowledge(vllm-omni): sync 19da2382 (PR #6054) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 10 +++++++++- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../model-executor/rules-platform-backends.md | 8 +++++++- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 2 +- knowledge/repos/vllm-omni/rebase/_index.md | 2 +- knowledge/repos/vllm-omni/rebase/workflow.md | 2 +- 13 files changed, 28 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ede059103..b6bacb5c3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: da4a08b65bfe717d8c4bc1a98decd0c8ad88976b + previous_audited_sha: 651b03225b69fe825525e1cfa00300fe856fcea0 audited_ref: main - audited_sha: 651b03225b69fe825525e1cfa00300fe856fcea0 + audited_sha: 19da23824637e76bdbb2e8131ec50eaf68903097 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 83b6d6542..f14470624 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -651b03225b69fe825525e1cfa00300fe856fcea0`. The canonical machine baseline +19da23824637e76bdbb2e8131ec50eaf68903097`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index a116f8c5a..c8ca7c9e4 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054"] --- # vLLM-Omni CI @@ -56,6 +56,14 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie L4 full-model run was deferred to Buildkite, so do not record a full test pass from this change alone. ^[PR #6303 / issue #6022] +## NPU diffusion-attention selector regression boundary + +- `tests/platforms/npu/test_diffusion_attn_backend_selector.py` is CPU-marked and loads the + NPU platform module with fake vLLM/vLLM-Ascend dependencies. It isolates eager `mindiesd` + import selection for FLASH, hub FLASH fallbacks, RAINFUSION, and SDPA plus missing-package + fallback behavior. It is selector regression coverage only: it neither requires an NPU or + MindIE-SD install nor establishes CANN registration or real Ascend-kernel execution. ^[PR #6054] + ## Qwen-Image nightly coverage and step-execution perf sharing - The Qwen-Image online-serving expansion keeps nine feature pytest IDs, but assigns one diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5c8ba73f4..edd654dac 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 651b0322` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 19da2382` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b4ae0c78a..323cfea03 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 651b0322` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 19da2382` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2697d0480..2061a0a89 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 651b0322` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 19da2382` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e58193709..f53112176 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 651b0322` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 19da2382` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index a7777ebf9..f379931ed 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102"] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102", "PR #6054", "vllm_omni/platforms/npu/platform.py", tests/platforms/npu/test_diffusion_attn_backend_selector.py] confidence: high --- @@ -84,3 +84,9 @@ confidence: high - 禁止：在 common adapter 直接导入 `vllm_ascend` 或硬编码 CUDA `BlockTables`/metadata 假设；把 platform capability 延迟到首次 attention dispatch；平台未提供 native contract 时静默回退 dense 或继续占用 paged KV。 - 验收：default 与 NPU hook 的 class、metadata 参数和 backend selection 通过 mock/contract 测试，unsupported hook/backend 在物理 cache allocation 前 fail fast；CUDA 与 NPU lane 分别验证 native metadata construction，未运行真实 NPU 硬件时不得宣称 Ascend 生产 parity。^[PR #6102] +## EXEC-13f — NPU diffusion attention 必须只为会触达 MindIE-SD 的后端提前导入 + +- 触发：修改 `NPUOmniPlatform.get_diffusion_attn_backend_cls()` 的 diffusion attention backend 选择、NPU FlashAttention/RAINFUSION fallback，或 MindIE-SD custom-op 环境初始化。 +- 强制：显式 `FLASH_ATTN`（包括在 NPU 上回退到本地 FlashAttention 的 `FLASH_ATTN_HUB` / `FLASH_ATTN_3_HUB`）和 `RAINFUSION_ATTN` 在 `mindiesd` 可发现时必须在 backend path 返回前导入它；后者的 dense FlashAttention fallback 会在 `start_step` 前或没有可稀疏化 video segment 的 layer 触达 MindIE-SD。该导入须早于任何会让 CANN 固化 custom-op registry 的 regInfo lookup，使 `mindiesd.env` 能先把 vendor dirs 写入 `ASCEND_CUSTOM_OPP_PATH`。 +- 禁止：为 `TORCH_SDPA` 或其他不会到达 MindIE-SD kernel 的显式 backend 导入可选 `mindiesd`；不得把 optional-package 损坏扩散成这些 backend 的选择失败，也不得假定所有 RAINFUSION layer 都走 sparse path。 +- 验收：CPU/mock selector test 应分别断言 FLASH、两个 hub FLASH fallback 与 RAINFUSION 触发 eager import，TORCH_SDPA 不触发；同时覆盖 `mindiesd` 缺失时显式 FLASH 仍解析、未选择 backend 时才在可用条件下默认 FLASH，否则回退 SDPA。该测试不运行 NPU/CANN，不证明 custom-op 注册、真实 kernel 或 Ascend 生产可用性。^[PR #6054] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index abdf335d7..6651653c5 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 651b0322` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 19da2382` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8219eb277..cf42cc43b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 651b0322` 验证存在 +- 源码校验：以上路径均已在 `main @ 19da2382` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index de4f77fcd..d9a1bcf0e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 651b0322`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 19da2382`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 diff --git a/knowledge/repos/vllm-omni/rebase/_index.md b/knowledge/repos/vllm-omni/rebase/_index.md index 5d357127d..0357eceaa 100644 --- a/knowledge/repos/vllm-omni/rebase/_index.md +++ b/knowledge/repos/vllm-omni/rebase/_index.md @@ -10,7 +10,7 @@ sources: ["PR #6606", ".buildkite/cuda/rebase-pipeline.yml", "vllm-omni-rebase-a # Rebase（对齐 upstream vLLM） 把 vllm-omni 对齐到新版 upstream vLLM 的周期性工作：专用分支 `dev/vllm-align`、 -专用管线 `.buildkite/cuda/rebase-pipeline.yml`（仓库 `main @ 651b0322`、upstream vLLM +专用管线 `.buildkite/cuda/rebase-pipeline.yml`（仓库 `main @ 19da2382`、upstream vLLM `v0.28.0 @ 2cf0a6915ce5` 验证）、专职 自动化（rebase-agent 仓库——运营系统在那里，本主题只沉淀领域知识）。 diff --git a/knowledge/repos/vllm-omni/rebase/workflow.md b/knowledge/repos/vllm-omni/rebase/workflow.md index 758a1c7e5..b57c88390 100644 --- a/knowledge/repos/vllm-omni/rebase/workflow.md +++ b/knowledge/repos/vllm-omni/rebase/workflow.md @@ -10,7 +10,7 @@ sources: ["PR #6606", "vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-o # Rebase 工作流：分支、波次与失败路由 运营事实来自 rebase-agent 配置快照（@122a9468，**可能漂移**）；仓库侧事实在 -`main @ 651b0322` 复核。运营系统以 rebase-agent 仓库为准，本页是知识树快照； +`main @ 19da2382` 复核。运营系统以 rebase-agent 仓库为准，本页是知识树快照； upstream release baseline 为 `v0.28.0 @ 2cf0a6915ce5`。^[PR #6606] ## 分支与管线 From 0d944741bd363e449ed3f992b797a82bdcc842a1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:26:16 +0800 Subject: [PATCH 330/465] knowledge(vllm-omni): sync 94b1546c (PR #6284) --- adapters/vllm_omni/release_baseline.yaml | 8 ++++---- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++++---- .../repos/vllm-omni/models/qwen-omni/_index.md | 12 +++++++----- .../vllm-omni/models/qwen-omni/architecture.md | 10 ++++++++-- .../repos/vllm-omni/models/qwen-omni/rules.md | 18 +++++++++++++++++- 12 files changed, 47 insertions(+), 23 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b6bacb5c3..4462e76f7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 651b03225b69fe825525e1cfa00300fe856fcea0 + previous_audited_sha: 19da23824637e76bdbb2e8131ec50eaf68903097 audited_ref: main - audited_sha: 19da23824637e76bdbb2e8131ec50eaf68903097 + audited_sha: 94b1546ce3a04a0b0f7c0a685e1db2af05d5be7a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -16,8 +16,8 @@ inventories: count: 65 sha256: 6107384ad499e668c1ed3690d740b7be816c900c504f244dce557ef51a257f2a pipelines: - count: 55 - sha256: 9d5033c9a7ffd7ff636e32db0f9be2a549b7c16b034e13817e280f19a9e16769 + count: 56 + sha256: 39e0a741462b11a2c96fa809be8ea88d502fbb4bc9068ab1a74ea6d7c6c6f1d6 deploy_yamls: count: 85 sha256: 205c0e5c20888852e8abb4b83d8fa81cc59066c7010f3db374f889f879ab6160 diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f14470624..51ba144d4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -19da23824637e76bdbb2e8131ec50eaf68903097`. The canonical machine baseline +94b1546ce3a04a0b0f7c0a685e1db2af05d5be7a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index edd654dac..5f383a34f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 19da2382` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 94b1546c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 323cfea03..a83e5cd2f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 19da2382` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 94b1546c` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2061a0a89..89536ef46 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 19da2382` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 94b1546c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f53112176..9a476112d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 19da2382` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 94b1546c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 6651653c5..cb994fe18 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 19da2382` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 94b1546c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cf42cc43b..b77da74f0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 19da2382` 验证存在 +- 源码校验：以上路径均已在 `main @ 94b1546c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d9a1bcf0e..19293c029 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 19da2382`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 94b1546c`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -37,7 +37,7 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 |---|---|---| | AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | | Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 40 个模型族目录 | -| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 55 个 key | +| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 56 个 key | | Deploy YAML | `vllm_omni/deploy/*.yaml` | 85 份 | PR #6089 新增 `nemotron_labs_voicechat_duplex.yaml`，因此只增加 deploy profile；它没有 @@ -90,7 +90,7 @@ lingbot_video、lingbot_world、longcat_image、longcat_video、ltx2、ming_flas minimax_h3、nextstep_1_1、omnigen2、omnivoice、ovis_image、pi0、qwen_image、sana_video、sana_wm、sd3、 sdxl、sensenova_u1、stable_audio、wan2_2、z_image -## OMNI_PIPELINES key（55） +## OMNI_PIPELINES key（56） Gr00tN1d7（注意:唯一 CamelCase key）、audex_s2s、audex_thinker_only、audex_tta、 audex_tts、aura_omni、bagel、bagel_single_stage、bagel_think、cosyvoice3、covo_audio、 @@ -101,7 +101,7 @@ ming_flash_omni、ming_flash_omni_image、ming_flash_omni_thinker_only、 ming_flash_omni_tts、ming_tts、ming_tts_moe、minicpmo_4_5、minimax_music3、 moss_tts_delay、moss_tts_local、moss_tts_nano、moss_tts_realtime、nemotron_labs_audex、 nemotron_labs_voicechat、nemotron_voicechat、omnivoice、personaplex、pi0、qwen2_5_omni、 -qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_tts、step_audio_2、 +qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_omni_moe_thinker_only、qwen3_tts、step_audio_2、 step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v 注意：单 stage diffusion 模型**多数不在** `OMNI_PIPELINES`（引擎为它们生成 diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index d8fce4893..1cb0255fc 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -4,20 +4,22 @@ created: 2026-07-16 updated: 2026-09-04 type: index tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/quantization/component_config.py, docs/design/qwen3_omni_tts_performance_optimization.md] +sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/quantization/component_config.py, docs/design/qwen3_omni_tts_performance_optimization.md] --- # Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS） -- 常见别名：`qwen2_5_omni`、`qwen3_omni_moe`、`qwen3_tts`（家族目录：同一 thinker/ +- 常见别名：`qwen2_5_omni`、`qwen3_omni_moe`、`qwen3_omni_moe_thinker_only`、`qwen3_tts`（家族目录：同一 thinker/ talker/code2wav 谱系的多个 checkpoint/代际，按别名规则共用本目录） -- 源码模型族（`main @ e04210d6` 验证）：`model_executor/models/qwen2_5_omni/`、 +- 源码模型族（`main @ 94b1546c` 验证）：`model_executor/models/qwen2_5_omni/`、 `qwen3_omni/`、`qwen3_tts/`；pipeline registry key `qwen2_5_omni`、 `qwen2_5_omni_thinker_only`、`qwen3_omni_moe`（resolver `resolve_qwen3_omni_pipeline`，位于 `models/qwen3_omni/pipeline.py`，由 - `pipeline_registry.py:86/:100` 引用）、`qwen3_tts` + `pipeline_registry.py:86/:100` 引用）、`qwen3_omni_moe_thinker_only`（静态单 stage + `PipelineConfig`，绕过 resolver）、`qwen3_tts` - deploy YAML：`qwen2_5_omni.yaml`（1×H100 验证）、`qwen3_omni_moe.yaml` - （2×H100 验证）、`qwen3_omni_moe_mori_intranode.yaml`、`qwen3_tts.yaml` + （2×H100 验证）、`qwen3_omni_moe_mori_intranode.yaml`、`qwen3_omni_moe_thinking.yaml` + （显式 thinker-only 覆盖）与 `qwen3_tts.yaml` （+ `_forced_aligner`、`_high_concurrency` 变体） - Qwen2.5 thinker-only 的 ModelOpt NVFP4 W4A4 checkpoint 支持、stage sub-config、Talker ingress/internal width 与硬件/accuracy 证据边界归下方 architecture 入口。 diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md index 7c3cf492a..137f65952 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md @@ -4,12 +4,12 @@ created: 2026-07-16 updated: 2026-09-04 type: architecture tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", "PR #5671", docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/config/model.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/registry.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/worker/gpu_model_runner.py] +sources: ["PR #5073", "PR #5671", "PR #6284", docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/config/model.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/registry.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/worker/gpu_model_runner.py] --- # Qwen-Omni 家族拓扑与性能结论 -以下事实在 `main @ e04210d6` 复核。 +以下事实在 `main @ 94b1546c` 复核。 ## Stage 拓扑 @@ -25,6 +25,12 @@ sources: ["PR #5073", "PR #5671", docs/design/qwen3_omni_tts_performance_optimiz - Qwen3-Omni 的 pipeline 由 resolver（`models/qwen3_omni/pipeline.py:: resolve_qwen3_omni_pipeline`）按 checkpoint 结构动态决定，而不是冻结的 `PipelineConfig` 字面量。 +- `qwen3_omni_moe_thinker_only` 是例外：它直接注册冻结的单 Thinker、text→text + `PipelineConfig`。`qwen3_omni_moe_thinking.yaml` 显式指定该 key，所以 Instruct + checkpoint 即使由 HF `enable_audio_output` 判为全三 stage，也会加载 Thinker 而不加载 + Talker/Code2Wav；Captioner/Thinking checkpoint 不带该 YAML 时仍由 resolver 自动选同一 + 单 stage 拓扑。该变更只证明配置解析和拓扑选择，未提供质量、吞吐、显存或端到端 serve + 证据。^[PR #6284] ## Qwen2.5-Omni ModelOpt NVFP4 边界 diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index 903052056..e00dac0e2 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5687", "PR #6449", vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py] +sources: ["PR #5687", "PR #6284", "PR #6449", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py] confidence: high --- @@ -31,3 +31,19 @@ confidence: high - 验收：以真实 component-wrapped `CompressedTensorsConfig`（含 `None` component 与 default） 断言 `ignore` HF names 映射到最终路径；加载 AWQ checkpoint 时三 stage 都能完成 weight loading，且不出现缺模块/参数或 routed-expert quantized-weight 属性错误。^[PR #5687] + +## QOMNI-1c — Instruct 的 thinker-only deploy 必须选择静态 pipeline key + +- 触发：修改 Qwen3-Omni 的 pipeline registry、`qwen3_omni_moe_thinking.yaml`，或依赖 + `enable_audio_output` 自动拓扑选择的 serve/config-factory 路径。 +- 强制：需要 Instruct 权重只加载 Thinker 时，deploy YAML 必须显式设 + `pipeline: qwen3_omni_moe_thinker_only`；该 registry key 必须直接绑定静态单 stage + text→text `PipelineConfig`，不能重新进入由 HF config 决定全三 stage 的 + `qwen3_omni_moe` resolver。没有显式 YAML 的 Captioner/Thinking checkpoint 继续由 resolver + 自动选择 thinker-only 拓扑。 +- 禁止：把 Instruct 的 `enable_audio_output` 当作 thinker-only 的充分条件；只修改 YAML 而未 + 注册静态 key；或以 key 存在/原始 YAML 替代最终 resolved pipeline 的断言。 +- 验收：分别以 Instruct 和 Thinking/Captioner HF config 经 `StageConfigFactory` 加载该 YAML， + 断言同一 `qwen3_omni_moe_thinker_only` pipeline、仅一个 Thinker stage 和 text output；同时 + 直接解析该 key，断言其对象身份为 registry 的静态 `PipelineConfig` 而非 resolver 结果。PR + 只有这类配置单元测试，未提供真实 `vllm serve`、质量、性能或资源证据。^[PR #6284] From d6bf64794ce820e50502d65e03e78761612cb40c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:39:38 +0800 Subject: [PATCH 331/465] knowledge(vllm-omni): sync c6df39e9 (PR #6634) --- adapters/vllm_omni/manifest.yaml | 5 ++ adapters/vllm_omni/release_baseline.yaml | 6 +- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/benchmark/_index.md | 4 +- .../vllm-omni/benchmark/guides/_index.md | 3 +- .../benchmark/guides/omni-duplex-eval.md | 74 +++++++++++++++++++ .../repos/vllm-omni/benchmark/overview.md | 11 ++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 10 ++- 14 files changed, 108 insertions(+), 19 deletions(-) create mode 100644 knowledge/repos/vllm-omni/benchmark/guides/omni-duplex-eval.md diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index b162524d1..d32209732 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -163,6 +163,11 @@ modules: test_paths: [tests/benchmarks/test_serve_cli.py] import_check: "from vllm_omni.benchmarks.serve import main; from vllm_omni.entrypoints.cli.benchmark.serve import OmniBenchmarkServingSubcommand; print('Benchmarks imports OK')" wave: 2 + fullduplex: + local_paths: [vllm_omni/experimental/fullduplex/] + test_paths: [tests/benchmarks/duplex/, tests/e2e/features/fullduplex/] + import_check: "from vllm_omni.experimental.fullduplex.client import RealtimeDuplexClient; print('Full-duplex client imports OK')" + wave: 2 review_routes: - prefix: vllm_omni/patch.py diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4462e76f7..b4be865fb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 19da23824637e76bdbb2e8131ec50eaf68903097 + previous_audited_sha: 94b1546ce3a04a0b0f7c0a685e1db2af05d5be7a audited_ref: main - audited_sha: 94b1546ce3a04a0b0f7c0a685e1db2af05d5be7a + audited_sha: c6df39e9f7ad8a12fd61bc2c5007103c6a62076a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. @@ -135,8 +135,6 @@ ignored_paths: reason: legal metadata - pattern: README.md reason: repository overview -- pattern: realtime_video_prompt_interaction_protocol.md - reason: deleted upstream documentation artifact - pattern: SECURITY.md reason: community metadata - pattern: setup.py diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 51ba144d4..6d2540a77 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -94b1546ce3a04a0b0f7c0a685e1db2af05d5be7a`. The canonical machine baseline +c6df39e9f7ad8a12fd61bc2c5007103c6a62076a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/benchmark/_index.md b/knowledge/repos/vllm-omni/benchmark/_index.md index 05c1a7cd1..1f12adf69 100644 --- a/knowledge/repos/vllm-omni/benchmark/_index.md +++ b/knowledge/repos/vllm-omni/benchmark/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, benchmark] -sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml, "PR #6522", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py] +sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml, "PR #6522", "PR #6634", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/benchmarks/duplex/, vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py, vllm_omni/experimental/fullduplex/client.py] --- # vLLM-Omni 性能与 Profiling @@ -22,7 +22,7 @@ sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml, "PR #6 | 遇到什么 | 查看哪里 | |---|---| | 查看旧 benchmark 总览 | [overview](overview.md) | -| 运行 AR graph 或 Hunyuan benchmark | [benchmark guides](guides/_index.md) | +| 运行 AR graph、Hunyuan 或 MiniCPM-o Omni-DuplexEval benchmark | [benchmark guides](guides/_index.md) | | 调查 profiling、模型加载和性能验证错误 | [benchmark incidents](incidents/_index.md) | | 查询历史结果 | [results](results/_index.md) | | 运行/审查 OmniInteract native-duplex serving benchmark | [OmniInteract realtime guide](guides/omniinteract-realtime.md) | diff --git a/knowledge/repos/vllm-omni/benchmark/guides/_index.md b/knowledge/repos/vllm-omni/benchmark/guides/_index.md index e11f3cf82..fb4c77bf7 100644 --- a/knowledge/repos/vllm-omni/benchmark/guides/_index.md +++ b/knowledge/repos/vllm-omni/benchmark/guides/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, benchmark] -sources: ["PR #6522", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py] +sources: ["PR #6522", "PR #6634", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/duplex/] --- # vLLM-Omni 性能指南 @@ -24,3 +24,4 @@ sources: ["PR #6522", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, | 验证分叉执行路径 | [divergent path](divergent-path-validation.md) | | 查看旧 plan/validation 总入口 | [plan and validation](plan-and-validation.md) | | 复现或审查 OmniInteract realtime serving runner | [OmniInteract realtime](omniinteract-realtime.md) | +| 生成、评分或审查 MiniCPM-o Omni-DuplexEval artifact | [Omni-DuplexEval](omni-duplex-eval.md) | diff --git a/knowledge/repos/vllm-omni/benchmark/guides/omni-duplex-eval.md b/knowledge/repos/vllm-omni/benchmark/guides/omni-duplex-eval.md new file mode 100644 index 000000000..5d44919f8 --- /dev/null +++ b/knowledge/repos/vllm-omni/benchmark/guides/omni-duplex-eval.md @@ -0,0 +1,74 @@ +--- +title: "MiniCPM-o Omni-DuplexEval" +created: 2026-09-04 +updated: 2026-09-04 +type: guide +tags: [vllm-omni, benchmark] +sources: ["PR #6634", examples/online_serving/minicpmo/README.md, vllm_omni/benchmarks/duplex/, vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py, vllm_omni/experimental/fullduplex/client.py] +confidence: high +--- + +# MiniCPM-o Omni-DuplexEval + +`vllm bench omni-duplex-eval --omni ` is the +installed workflow for evaluating MiniCPM-o native-duplex output. Generation +uses the vLLM-Omni WebSocket endpoint; evaluation uses a separate +OpenAI-compatible multimodal judge. This exercises vLLM-Omni's duplex client, +artifact, and local-judge integration. It does not reproduce the paper's +original inference implementation or establish paper-score parity. ^[PR #6634] + +## Selection and generation contract + +- `Hothan/Omni-DuplexEval` contains six RTD and three PR Hugging Face splits; + those names are splits, not dataset configurations. `--split` selects one, + while omitted/`all` loads all nine. Local JSON/JSONL manifests and + `--media-root` are supported; `--ids` and `--limit` filter after normalized + samples are built. +- PR task semantics come from the split name when a row has no explicit task: + correction, event reminder, or post-event reminder. Do not silently default + an unknown split to a reminder task. +- Generation currently supports only `--mix question`. It converts question + audio to mono 16-kHz PCM16 when needed, interleaves optional JPEG frames, + and streams media units through `RealtimeDuplexClient`. Realtime pacing is + required for scoreable media-clock timing; as-fast-as-possible artifacts are + marked `clock=invalid`. + +## Lifecycle and artifact contract + +- The client periodically acknowledges playback while streaming, commits the + input, waits for `response.done`, acknowledges the remaining playback, and + closes the session. A response or close timeout is retained in metadata + rather than erased; `response_done` therefore remains distinct from merely + having a response file. +- Each generated sample writes `//.json` plus a + sibling `.meta.json`. Metadata pins pace, clock, mix, FPS, unit size, model, + response/timeout state, and the reference-audio SHA-256. Evaluation rejects + `clock=invalid` unless `--allow-invalid-clock` is explicit. +- Text deltas are sentence-split and timestamped against the input media clock. + Every accepted millisecond clock field is divided by 1000; wall-clock and + media-clock values must not be mixed. + +## Scoring protocol + +- Prompt wording and window math are pinned by `PROTOCOL_PIN` to + `OpenBMB/Omni-DuplexEval@ca3c122b4d4bf67afd6b18ea5e724b4561bdde48`. + Changing either requires a pin bump and a recorded score delta. +- For RTD temporal scoring, a response sentence uses + `start=max(0, sentence_start-2)` and + `end=max(start+0.5, sentence_end-2)`. This keeps early responses instead of + discarding negative pre-roll. Content scoring uses the full video URL or + explicitly sampled frames; the frame-sample path must pass those frames to + the content judge, not only to temporal scoring. +- PR correction scores the complete response. Reminder tasks score response + text in each event's configurable post-start window, and a sample succeeds + only when every required event succeeds. `summarize` aggregates RTD content + and temporal means by split and PR all-event success by task. + +## Evidence boundary + +The PR author reported a stratified 27-case run—three samples from each of the +nine splits—with all generation and judge jobs completing. That is a smoke of +the 660-case dataset, not a full evaluation. The reported scores used +vLLM-Omni MiniCPM-o plus a Qwen2.5-VL-7B frame-sample judge, so they are +environment- and judge-specific; do not turn them into quality, latency, full +coverage, or paper-equivalence guarantees. ^[PR #6634] diff --git a/knowledge/repos/vllm-omni/benchmark/overview.md b/knowledge/repos/vllm-omni/benchmark/overview.md index d4b96b0eb..a6ecbf720 100644 --- a/knowledge/repos/vllm-omni/benchmark/overview.md +++ b/knowledge/repos/vllm-omni/benchmark/overview.md @@ -1,10 +1,10 @@ --- title: "Benchmark Notes" created: 2026-07-10 -updated: 2026-07-10 +updated: 2026-09-04 type: guide tags: [vllm-omni, benchmark] -sources: [] +sources: ["PR #6634", vllm_omni/benchmarks/duplex/, vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py, vllm_omni/experimental/fullduplex/client.py] --- # Benchmark Notes @@ -16,4 +16,11 @@ sources: [] | [ar_2025_04_17.md](results/ar-2025-04-17.md) | 2025-04-17 HunyuanImage3 AR HF baseline vs vLLM-Omni 对齐 benchmark 结论 | | [AR graph online 结果](results/ar-graph-online-2026-06-09-overview.md) | 2026-06-09/06-12 HunyuanImage3 AR graph online perf、trace、patch 对照和 PR 证据口径 | +## Omni-DuplexEval + +PR #6634 introduced one installed CLI with `generate`、`evaluate` and `summarize` +subcommands for MiniCPM-o native-duplex artifacts. Its dataset, media-clock, +scoring-protocol and evidence contracts are documented in the +[Omni-DuplexEval guide](guides/omni-duplex-eval.md). ^[PR #6634] + 具体 PR 的临时复测、讲解稿和 PR body 数据不再作为长期 benchmark 文档保留。这里仅路由可跨 PR 复用的 workload、口径、artifact 和诊断经验。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5f383a34f..54dac269e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 94b1546c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c6df39e9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a83e5cd2f..92519f868 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 94b1546c` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c6df39e9` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 89536ef46..e5842411c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 94b1546c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c6df39e9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9a476112d..3083efeba 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 94b1546c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c6df39e9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cb994fe18..42e65955e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 94b1546c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c6df39e9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b77da74f0..25f6dcaa6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 94b1546c` 验证存在 +- 源码校验：以上路径均已在 `main @ c6df39e9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 19293c029..485a1ab73 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 94b1546c`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c6df39e9`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,6 +45,10 @@ PR #6089 新增 `nemotron_labs_voicechat_duplex.yaml`，因此只增加 deploy p Nemotron VoiceChat 的单会话 eager fp32 配置，不能从 YAML 或 PR 的作者测试外推 wall-clock realtime、确定性 barge-in 或一般硬件支持。^[PR #6089] +PR #6634 adds Omni-DuplexEval benchmark support without changing the AR registry, diffusion +registry, `OMNI_PIPELINES`, or deploy-YAML inventory. Its MiniCPM-o duplex endpoint is an +evaluation consumer, not a new model registration or deploy profile. ^[PR #6634] + 对比上一审计快照（`39c16d75`）：AR 架构 88→81，diffusion pipeline 65→64， OMNI_PIPELINES 57→55，deploy 86→84。PR #6353 删除 AudioX 的 diffusion registry、 processor/extras/recipe/examples/tests，并删除 MammothModa2 Preview/Dev 共用的七个 AR @@ -113,8 +117,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 39c16d75 \ - --to a94b2580 \ + --from 94b1546c \ + --to c6df39e9 \ --repo \ --mode report-only ``` From d1fa468ca8c573356311e1832eea628a38829bb8 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:44:37 +0800 Subject: [PATCH 332/465] knowledge(vllm-omni): sync 54c1fc64 (PR #6680) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/configuration/rules.md | 4 ++-- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/rules.md | 6 +++--- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b4be865fb..24592186e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 94b1546ce3a04a0b0f7c0a685e1db2af05d5be7a + previous_audited_sha: c6df39e9f7ad8a12fd61bc2c5007103c6a62076a audited_ref: main - audited_sha: c6df39e9f7ad8a12fd61bc2c5007103c6a62076a + audited_sha: 54c1fc64b78ad6fe80c565ba9a8b8374d509224c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6d2540a77..c06665928 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c6df39e9f7ad8a12fd61bc2c5007103c6a62076a`. The canonical machine baseline +54c1fc64b78ad6fe80c565ba9a8b8374d509224c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 54dac269e..108ab5bd0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c6df39e9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 54c1fc64` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 110156632..c338ee0f3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820", "PR #6619"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820", "PR #6619", "PR #6680"] --- # vLLM-Omni 配置开发门禁 @@ -51,7 +51,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - **VOMNI-CFG-1d — 兼容修复不能自动继承原 scope。** 新修复若引入新的 owner、来源、优先级、转换或模型/服务路由，或实现量级接近预估两倍，必须先重切片；禁止按入口堆特例和重复测试。多 PR RFC 只实现当前 PR 的 merge condition，后续切片或顺手修复的行为连同专属测试一起 `DELETE / DEFER`，不能因为已经写完或测试通过就继承为当前 scope。验收时每个生产代码 hunk 都应属于本轮目的，同一语义跨入口使用参数化矩阵并保留一个 control。 - **VOMNI-CFG-1e — 严格配置固定按“归一化、验 owner、分区、构造”执行。** deploy/stage/CLI overlay、alias 和 flat→nested 转换必须先产出一份规范化映射；mixed CLI/orchestrator namespace 只允许先路由已明确声明的非 stage scope，剩余 stage 候选必须以完整 key 集合进入 ownership validator。validator 不能同时改值、路由字段或先丢 `None`；校验通过后的规范化结果就是后续分区和构造唯一可消费的 owner artifact，禁止只调用 validator 却继续传 raw mapping，也禁止末端重新做 alias、default、precedence 或静默过滤。structured、legacy startup 和仍公开可调用的 direct factory 必须复用同一组 normalization/validation primitive。 - **VOMNI-CFG-1f — key 是否已知与 value 是否有效必须分开判定。** 缺失、已知字段的 `None`、显式 `False`/`0`、未知字段的 `None` 和两个非空来源冲突是五种不同状态：只有明确 owner 可以把已知 `None` 解释为 unset，`False`/`0` 不得被 truthiness merge 吃掉，未知 key 即使为 `None` 也必须按 strict contract 报错，alias 只在 legacy 与 canonical 都提供非 `None` 值时冲突。验收至少覆盖本轮会改变分支的这些状态，并断言错误或最终 consumer。 -- **VOMNI-CFG-1g — rebase 或 schema 扩张后重新做配置归属审计。** 上游新增 dataclass 字段、默认 factory 参数、pipeline-wide 字段或 owner 集合变化时，把自动合并结果当作新的语义 diff；重新比较 base/head 的字段集合、序列化边界、默认-stage 透传和所有严格入口。默认 factory 的扩展字段应由 schema metadata 或单一声明集合驱动，禁止再加一份手写白名单。只有最新 base 上的真实 factory/legacy/structured 路径都保留合法非默认值并继续拒绝未知字段，才算 rebase 完成。 +- **VOMNI-CFG-1g — rebase 或 schema 扩张后重新做配置归属审计。** 上游新增 dataclass 字段、默认 factory 参数、pipeline-wide 字段或 owner 集合变化时，把自动合并结果当作新的语义 diff；重新比较 base/head 的字段集合、序列化边界、默认-stage 透传和所有严格入口。部署拥有的 lifecycle field 必须在 structured stage model config 与 legacy engine-args projection 中都显式写入，并由 `OmniEngineArgs.create_model_config()` 到最终 `OmniModelConfig`；默认值相同不能证明该投影存在。默认 factory 的扩展字段应由 schema metadata 或单一声明集合驱动，禁止再加一份手写白名单。只有最新 base 上的真实 factory/legacy/structured 路径都保留合法非默认值并继续拒绝未知字段，才算 rebase 完成。^[PR #6680] - **VOMNI-CFG-1h — typed engine projection 与生产 cutover 是两个 merge condition。** `build_engine_args_dict_from_omni_stage_config` 从 model/load/cache/scheduler/runtime/connector/ parallel/quantization/diffusion typed owner 深拷贝非 `None` 字段，再与 legacy 共用 model/tokenizer diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 92519f868..32b1dbeb2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c6df39e9` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 54c1fc64` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e5842411c..b00540fdf 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c6df39e9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 54c1fc64` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3083efeba..a7c948be6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c6df39e9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 54c1fc64` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 42e65955e..3eb5bdbea 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c6df39e9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 54c1fc64` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index b42aeb6ae..321f23c8b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619", "PR #6680"] --- # Scheduler 规则 @@ -267,9 +267,9 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 ## SCHED-5g — resumable async-chunk 终态清理必须以 live queue 所有权为准 - 触发：resumable async-chunk 请求在一个流式 segment 结束时进入 `FINISHED_STOPPED`，但仍由 `running`、`waiting`、`skipped_waiting` 或 connector 隐藏队列持有，随后 session close、取消或其他路径调用 `finish_requests`。 -- 强制：先物化可能被多层消费的 request-id iterator，并在 adapter 清理前快照 `skipped_waiting` 中承担流式等待计数的请求；只对仍有活队列所有权的目标 resumable 终态请求恢复状态，`skipped_waiting` 恢复为 `WAITING_FOR_STREAMING_REQ`，`running`/`waiting` 按实际队列对齐；下游 async-chunk 的 segment stop 必须先清除该 segment 的 finished 标记，再从 `skipped_waiting` 转为 ordinary `WAITING` 并恢复 connector polling，同一 update 尾部按 stale stopped 集合清理时不得移除这个已重新入队的 request；running purge 必须限定本次 finish 集合，并确保 `_free_request`、coordinator 与 connector 清理恰好执行一次。 +- 强制：先物化可能被多层消费的 request-id iterator，并在 adapter 清理前快照 `skipped_waiting` 中承担流式等待计数的请求；只对仍有活队列所有权的目标 resumable 终态请求恢复状态，`skipped_waiting` 恢复为 `WAITING_FOR_STREAMING_REQ`，`running`/`waiting` 按实际队列对齐；下游 async-chunk 的 segment stop 必须先清除该 segment 的 finished 标记。只有 connector `receives_chunks`、最终 stage `model_config.session_mode == "duplex"` 且 request 仍为 `WAITING_FOR_STREAMING_REQ` 时，才从 `skipped_waiting` 转为 ordinary `WAITING` 并恢复 connector polling；sender-only 或 turn-mode stage 保持 parked，等待显式 streaming update。同一 update 尾部按 stale stopped 集合清理时不得移除已重新入队的 request；running purge 必须限定本次 finish 集合，并确保 `_free_request`、coordinator 与 connector 清理恰好执行一次。 - 禁止：把任意已完成请求重新打开；对脱离所有 live queue、可能等待 deferred block free 的终态请求调用释放；因 request 在本轮进入时是 `WAITING_FOR_CHUNK` 就撤销其同轮 requeue；全局清空 running 中无关的 resumable segment；重复消费单遍 request-id iterator，或因非流式 skipped 请求错误减少 streaming counter。 -- 验收：AR 与 generation scheduler 均覆盖 hidden、`running`、`waiting`、`skipped_waiting`、脱离队列及无关 resumable 请求；另覆盖下游 receiver 从 segment stop 同轮转回 `WAITING`，断言它保留在 waiting queue、离开 skipped queue、流式等待计数递减且 segment-finished 标记清除；首次 finish 恰好释放、第二次无操作，并验证 generator request-id 输入。^[PR #6089] ^[PR #6360] +- 验收：AR 与 generation scheduler 均覆盖 hidden、`running`、`waiting`、`skipped_waiting`、脱离队列及无关 resumable 请求；另覆盖 connector-fed duplex receiver 从 segment stop 同轮转回 `WAITING`，断言它保留在 waiting queue、离开 skipped queue、流式等待计数递减且 segment-finished 标记清除；sender-only duplex 与 connector-fed turn-mode controls 必须仍 parked，同时也清除标记。首次 finish 恰好释放、第二次无操作，并验证 generator request-id 输入。PR 作者报告的 H100 E2E 与 574-pass CPU/config suite 是提交时证据，不是当前环境或跨硬件保证。^[PR #6089] ^[PR #6360] ^[PR #6680] ## SCHED-6a — async discard 的计数单位必须与 stale drain 一致 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 25f6dcaa6..fba89020e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c6df39e9` 验证存在 +- 源码校验：以上路径均已在 `main @ 54c1fc64` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 485a1ab73..ccc758ef2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c6df39e9`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 54c1fc64`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -117,8 +117,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 94b1546c \ - --to c6df39e9 \ + --from c6df39e9 \ + --to 54c1fc64 \ --repo \ --mode report-only ``` From 5e4452483ebf8814af484c20ce4deaedce641cf1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:50:11 +0800 Subject: [PATCH 333/465] knowledge(vllm-omni): sync 069e6c5f (PR #6678) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md | 2 +- .../repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md | 9 ++++++++- 11 files changed, 21 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 24592186e..e79db1983 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c6df39e9f7ad8a12fd61bc2c5007103c6a62076a + previous_audited_sha: 54c1fc64b78ad6fe80c565ba9a8b8374d509224c audited_ref: main - audited_sha: 54c1fc64b78ad6fe80c565ba9a8b8374d509224c + audited_sha: 069e6c5fbc829d56490529df3c169c300a46043d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c06665928..dd0f20993 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -54c1fc64b78ad6fe80c565ba9a8b8374d509224c`. The canonical machine baseline +069e6c5fbc829d56490529df3c169c300a46043d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 108ab5bd0..d07392ee4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 54c1fc64` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 069e6c5f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 32b1dbeb2..8dc4c9b66 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 54c1fc64` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 069e6c5f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b00540fdf..85b1e69f8 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 54c1fc64` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 069e6c5f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a7c948be6..7567a41fa 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 54c1fc64` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 069e6c5f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3eb5bdbea..e7f89fee5 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 54c1fc64` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 069e6c5f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fba89020e..4519eab78 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 54c1fc64` 验证存在 +- 源码校验：以上路径均已在 `main @ 069e6c5f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ccc758ef2..af0ff6fbc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 54c1fc64`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 069e6c5f`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -117,8 +117,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from c6df39e9 \ - --to 54c1fc64 \ + --from 54c1fc64 \ + --to 069e6c5f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index e31ba5fb3..7c1614a4c 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082"] +sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6678", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082"] confidence: high --- diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md index 5a77e49bb..0735ecfa2 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6318", "PR #6346", "PR #6458", "PR #6619", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, tests/config/test_config_factory.py, tests/entrypoints/openai_api/test_duplex_handler.py] +sources: ["PR #6318", "PR #6346", "PR #6458", "PR #6619", "PR #6678", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] confidence: high --- @@ -42,3 +42,10 @@ confidence: high - 强制：所有 shipping profile 设置 `session_mode=duplex`、`active_stream_window=1`，且 `duplex_session.max_sessions` 与 Stage 0/1 的 `max_num_seqs` 容量一致；replica overlay 从 base 继承这些字段。YAML 保留 chat 的 Stage 1 `min_tokens=50` 与既有 codec 参数，native duplex adapter 仅在该 session 的 runtime config 覆盖 `min_tokens=0`。AR stage 保留默认 async scheduler。 - 禁止：恢复独立 `minicpmo_4_5_duplex.yaml`；在 replica overlay 重复并漂移 session 配置；为 duplex 全局关闭 async scheduling；或把 runtime-only floor 写回 YAML 而改变 chat TTS。 - 验收：展开四份 base profile 及三个 replica overlay，断言全部 duplex-enabled、窗口与 session capacity 正确；同一 server 分别验证 `/v1/realtime?duplex=1` 和 chat，前者 Stage 1 看到 `min_tokens=0`、后者仍看到 `50`。删除旧 overlay 是部署文件名迁移，外部显式引用必须同步更新。^[PR #6458] ^[PR #6619] + +- admission/expiry E2E probe 必须从实际启动的 deploy YAML 经 + `load_deploy_config(...).duplex_session.max_sessions` 取得 admission limit，不能保留 + 硬编码容量或在 test helper 重做 YAML merge/default/validation；这样 base-config overlay + 与未声明 `duplex_session` 时的 server 语义保持一致。CPU 守卫至少覆盖 shipping base 的 + 显式容量、未声明字段的 default，及一个从 base 继承容量的 overlay；live probe 再验证在 + 该 limit 后收到 capacity error。^[PR #6678] From a17cc3746d727df4ce0e0194f638528c47604843 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 22:54:57 +0800 Subject: [PATCH 334/465] knowledge(vllm-omni): sync 262ff146 (PR #6630) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/minicpm-o-4-5/rules-duplex.md | 10 +++++++++- 10 files changed, 21 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e79db1983..d56ec6819 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 54c1fc64b78ad6fe80c565ba9a8b8374d509224c + previous_audited_sha: 069e6c5fbc829d56490529df3c169c300a46043d audited_ref: main - audited_sha: 069e6c5fbc829d56490529df3c169c300a46043d + audited_sha: 262ff146935553d444702b583874da461b50c079 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index dd0f20993..eac104d1a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -069e6c5fbc829d56490529df3c169c300a46043d`. The canonical machine baseline +262ff146935553d444702b583874da461b50c079`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d07392ee4..b59ff244d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 069e6c5f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 262ff146` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8dc4c9b66..f5efec083 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 069e6c5f` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 262ff146` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 85b1e69f8..0ffcba626 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 069e6c5f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 262ff146` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7567a41fa..c7b8473c6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 069e6c5f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 262ff146` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e7f89fee5..cfcb917fa 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 069e6c5f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 262ff146` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4519eab78..1abe3b1b2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 069e6c5f` 验证存在 +- 源码校验：以上路径均已在 `main @ 262ff146` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index af0ff6fbc..15f72d50c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 069e6c5f`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 262ff146`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -117,8 +117,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 54c1fc64 \ - --to 069e6c5f \ + --from 069e6c5f \ + --to 262ff146 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md index 0735ecfa2..c79145034 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6318", "PR #6346", "PR #6458", "PR #6619", "PR #6678", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] +sources: ["PR #6318", "PR #6346", "PR #6458", "PR #6619", "PR #6630", "PR #6678", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] confidence: high --- @@ -36,6 +36,14 @@ confidence: high - 禁止：只依赖 YAML `max_tokens` 维持 turn 边界；改变 chat 的 whole-utterance Talker floor；在中间 chunk 永久屏蔽 EOS；把终止采样再次作为普通 codec frame 输出。 - 验收：覆盖 prefill 边界元数据、step 24/25 的 mask/force EOS、runtime Stage 1 `min_tokens=0`、chat sampling control、simplex 不误触发以及多请求 delta/finished 对齐。^[PR #6346] ^[PR #6619] +## MCPMO-4d — auto-response silence continuation 必须有硬性终止与所有权 fence + +- 触发：修改 MiniCPM-o native duplex auto-response 的 silence continuation、`force_listen`、LISTEN 事件或 response close。 +- 强制：auto-response 以 64 个 one-second continuation unit 为上限，普通 response 保持 8 个单位上限；前 63 个 auto payload 保持既有语义，第 64 个请求 `force_listen`。边界 LISTEN 不得再走 non-terminal auto reschedule，而必须结束当前 response 并保留 resumable Stage-0 request；模型忽略提示时，下一次 continuation 尝试不得 append，而发出 `response.listen` 与恰好一个 `response.done`。fallback 在跨 `send_json` await 前后必须同时验证 epoch 和 active response owner，防止 barge-in 打开的新 response 被旧 response 的终止路径关闭。 +- 禁止：让 auto-response 因 `force_listen` 仅为 logit hint 而无限续跑；在已达边界后继续 append；清除 resumable request；或在 await 后未复核 session/response 所有权即结束 response。 +- 验收：覆盖 64 次 append、最后一次 `force_listen`、第 65 次无 append 且仅一次 terminal protocol pair、边界 model LISTEN 的 terminal path，以及 LISTEN send 期间 barge-in/new epoch 不被 stale fallback 关闭。单元覆盖证明协议与 race fence；PR 的 H800 lifecycle run 混合了其他改动且有 `audio_clipped` artifact，不作为音频质量或性能证据。^[PR #6630] +- 已知边界：terminalization 保留 data-plane request，而 `active_response_id is None` 时不走 model-turn owner guard；同一 request/turn 的 late SPEAK 仍可能新建第二个 response。该 reviewer nit 在本提交中未修复，不能把“恰好一个 `response.done`”外推成后续迟到事件也被永久抑制；若补齐，需显式拒绝 bound 后的 late SPEAK 并覆盖回归。^[PR #6630] + ## MCPMO-4e — shipping profile 必须共服 chat 与 native duplex - 触发：修改任一 `minicpmo_4_5*.yaml`、duplex session placement 或 Stage 1 Talker 默认/运行时采样参数。 From 98b0bac7a20ba6d437567d8dc6bdbff98528ada9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:02:51 +0800 Subject: [PATCH 335/465] knowledge(vllm-omni): sync 5e4daaf1 (PR #6189) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../serving/rules-engine-lifecycle.md | 14 ++++++- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../repos/vllm-omni/models/ltx2/_index.md | 15 +++---- .../vllm-omni/models/ltx2/architecture.md | 11 +++-- .../repos/vllm-omni/models/ltx2/rules.md | 42 +++++++++++++++++++ 13 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/ltx2/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d56ec6819..f2173eec0 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 069e6c5fbc829d56490529df3c169c300a46043d + previous_audited_sha: 262ff146935553d444702b583874da461b50c079 audited_ref: main - audited_sha: 262ff146935553d444702b583874da461b50c079 + audited_sha: 5e4daaf1970ef230c50d6fb2c7ef8ed4895545f8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index eac104d1a..ded7418b1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -262ff146935553d444702b583874da461b50c079`. The canonical machine baseline +5e4daaf1970ef230c50d6fb2c7ef8ed4895545f8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b59ff244d..26f16889d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 262ff146` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5e4daaf1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f5efec083..97d7e1f33 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 262ff146` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 5e4daaf1` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0ffcba626..dd174a5ac 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 262ff146` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5e4daaf1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c7b8473c6..b75f65b21 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 262ff146` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5e4daaf1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cfcb917fa..62e404003 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 262ff146` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5e4daaf1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1abe3b1b2..5eb55f1b5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 262ff146` 验证存在 +- 源码校验：以上路径均已在 `main @ 5e4daaf1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 1396cfcc5..0811554bd 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5221", "Issue #4855", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, tests/engine/test_orchestrator_event_driven.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5221", "Issue #4855", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, tests/engine/test_orchestrator_event_driven.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py, "PR #6189", tests/entrypoints/test_async_omni_diffusion_config.py] confidence: high --- @@ -17,6 +17,18 @@ confidence: high ## Engine 生命周期合同 +### SERV-5r — default diffusion stage 必须保留并按显式性合并 model extras + +- 触发：修改 `AsyncOmniEngine` default diffusion stage fallback、promoted top-level model option、 + `stage_overrides` 或 local/unregistered Diffusers checkpoint 启动。 +- 强制：先保留 caller `extras`，只有 explicit non-`None` top-level promoted option 才覆盖同名 extra， + 其余键保持 caller value 或 builtin default；stage-0 `extras` override 必须合并进 default fallback 的 + 输入。registered stage 保持既有 complete override mapping，不得因修复 fallback 改写其语义。 +- 禁止：以 builtin default 覆盖 caller-only extra；让 unregistered/local checkpoint 忽略 stage-0 extras； + 只修 registered config 路径或只断言 helper dict 而不覆盖 fallback resolution。 +- 验收：覆盖 caller-only extras、explicit top-level precedence、stage-0 extras 在 default fallback 的 + end-to-end resolution，以及 registered path 不回归。^[PR #6189] + ### SERV-5a — sleep/wake 状态必须保留 stage 和 tag 作用域 - 触发：sleep/wake 接受 `stage_ids`、resource tags 或 partial wake。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 15f72d50c..c4dfef99d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 262ff146`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5e4daaf1`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -117,8 +117,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 069e6c5f \ - --to 262ff146 \ + --from 262ff146 \ + --to 5e4daaf1 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/ltx2/_index.md b/knowledge/repos/vllm-omni/models/ltx2/_index.md index 36b15c09d..d14511e1c 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/_index.md +++ b/knowledge/repos/vllm-omni/models/ltx2/_index.md @@ -1,16 +1,16 @@ --- -title: "LTX-2 家族（含 LTX-2.3）" +title: "LTX-2 家族（含 LTX-2.3/2.5）" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-04 type: index tags: [vllm-omni, models, ltx2] -sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, recipes/LTX/LTX-2.md] +sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, recipes/LTX/LTX-2.md, recipes/LTX/LTX-2.5.md] --- -# LTX-2 家族（含 LTX-2.3） +# LTX-2 家族（含 LTX-2.3/2.5） -- 常见别名：`LTX-2`、`LTX-2.3`、`ltx2`（LTX-2.3 是同一源码模块下的新 checkpoint/ - 版本，按 checkpoint 别名规则共用本目录） +- 常见别名：`LTX-2`、`LTX-2.3`、`LTX-2.5`、`ltx2`；这些 checkpoint 版本共用同一 + 源码模块，但 LTX-2.5 的视频 decode 默认使用 DiffVAE。 - 厂商/模型：Lightricks；22B 参数文本→视频+音频生成（T2V/I2V，48kHz 同步音频， 768x512 可达 20+ 秒）；Diffusers 格式 checkpoint `dg845/LTX-2.3-Diffusers` - 源码：`vllm_omni/diffusion/models/ltx2/`（纯 diffusion，无 AR stage） @@ -24,7 +24,7 @@ sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, rec ## 什么时候查这里 -- 问题只属于 LTX-2/2.3（pipeline、分辨率/帧数语义、graph 模式、性能基线）。 +- 问题只属于 LTX-2/2.3/2.5（pipeline、decoder、分辨率/帧数语义、graph 模式、性能基线）。 ## 不放什么 @@ -36,3 +36,4 @@ sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, rec | 遇到什么 | 查看哪里 | |---|---| | 模型结构、serving 方式与已有性能/精度证据 | [architecture](architecture.md) | +| LTX-2.5 DiffVAE 选择、artifact、并行或最小几何约束 | [LTX-2.5 decoder rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/ltx2/architecture.md b/knowledge/repos/vllm-omni/models/ltx2/architecture.md index a7562575d..1a5448380 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/architecture.md +++ b/knowledge/repos/vllm-omni/models/ltx2/architecture.md @@ -1,13 +1,13 @@ --- -title: "LTX-2/2.3 模型架构与证据索引" +title: "LTX-2/2.3/2.5 模型架构与证据索引" created: 2026-07-16 updated: 2026-09-04 type: architecture tags: [vllm-omni, models, ltx2] -sources: [recipes/LTX/LTX-2.md, vllm_omni/diffusion/registry.py, "#4381", "#4464", "PR #6342", vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_vae.py, tests/diffusion/models/ltx2/test_ltx2_vocoder_cuda.py, tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py, tests/e2e/accuracy/ltx/test_ltx_official_similarity.py] +sources: [recipes/LTX/LTX-2.md, recipes/LTX/LTX-2.5.md, vllm_omni/diffusion/registry.py, "#4381", "#4464", "PR #6189", "PR #6342", vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_vae.py, tests/diffusion/models/ltx2/test_ltx2_vocoder_cuda.py, tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py, tests/e2e/accuracy/ltx/test_ltx_official_similarity.py] --- -# LTX-2/2.3 模型架构与证据索引 +# LTX-2/2.3/2.5 模型架构与证据索引 以下事实在 `v0.26.0 @ a4ea67a2` 复核（合并后的 recipe 与 live registry）。 @@ -23,6 +23,11 @@ sources: [recipes/LTX/LTX-2.md, vllm_omni/diffusion/registry.py, "#4381", "#4464 `LTX2DistilledPipeline`，DMD2 使用 `LTX2T2VDMD2Pipeline`/ `LTX2I2VDMD2Pipeline`；T2V/I2V 通过是否提供 `image=` 选择，不再使用单独的 `*ImageToVideoPipeline` registry names。 +- LTX-2.5 的 video decode 将 canonical Native checkpoint 中的 decoder tensors 转换到 + Diffusers-compatible DiffVAE；full 与 distilled 的 one-/two-stage profile 默认将其作为额外 + video decoder component。ConvVAE 保留给 I2V encoding，也可由 startup-only + `ltx2_use_conv_vae=true` 显式选择作视频 decoder；具体 fail-closed 约束见 + [LTX-2.5 decoder rules](rules.md)。 - Python `forward` 只允许 `req` 作为 positional argument，其他参数必须 keyword-only； 这对直接调用者和显式 `--model-class-name` 覆盖是 breaking change，CLI/HTTP recipe 已按 named fields 调用。 diff --git a/knowledge/repos/vllm-omni/models/ltx2/rules.md b/knowledge/repos/vllm-omni/models/ltx2/rules.md new file mode 100644 index 000000000..023e8ed87 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/ltx2/rules.md @@ -0,0 +1,42 @@ +--- +title: "LTX-2.5 DiffVAE 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, ltx2, diffusion] +sources: ["PR #6189", recipes/LTX/LTX-2.5.md, vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_latents.py, vllm_omni/diffusion/models/ltx2/ltx2_request.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_vae.py] +confidence: high +--- + +# LTX-2.5 DiffVAE 规则 + +### LTX25-1 — LTX-2.5 的 decoder、统计量与 artifact 必须作为同一版本化合同维护 + +- 触发：修改 LTX-2.5 full/distilled one- or two-stage pipeline 的视频 decoder、`ltx2_use_conv_vae`、 + latent normalization、Native sidecar 或 component residency。 +- 强制：LTX-2.5 默认构造 canonical Native DiffVAE；仅布尔 `ltx2_use_conv_vae=true` 可在启动时 + opt into ConvVAE，I2V encoding 仍使用 ConvVAE。provided latent 的 normalize 与 decode 前的 + denormalize 必须从实际视频 decoder 取得 mean、std 和 scaling factor。Native decoder 与独立 + sidecar 先查 model root；Hub fallback 使用 artifact repository 自己的 pinned revision，不能把 + Diffusers model revision 跨仓库复用。DiffVAE 初始化必须要求 runtime `kernels==0.14.1`、支持的 + NATTEN Hub kernel 和可操作的失败说明。 +- 禁止：让 ConvVAE statistics 处理 DiffVAE latent；将非布尔 opt-in 静默转换；把 local primary model + path 误当成独立 sidecar 必须离线；使用未 pin 的 canonical Native artifact，或向用户建议不存在的 + production FlexAttention fallback。 +- 验收：覆盖 full/distilled profile 的默认和 ConvVAE opt-in、非布尔拒绝、Native decoder strict + conversion、active-decoder statistics、local-first/repository-scoped revision routing，以及缺少 NATTEN + runtime prerequisite 的 actionable error。^[PR #6189] + +### LTX25-2 — DiffVAE 请求边界与分布式 decode 必须显式 fail-closed + +- 触发：修改 LTX-2.5 decode、`vae_parallel_mode`、tiling、request resolution validation 或 request RNG。 +- 强制：DiffVAE request validation 从首个 decoder-stage kernel 推导最小 latent H/W，并在 denoising 前 + 拒绝不足的 geometry；ConvVAE opt-in 保留其原有边界。分布式 DiffVAE 只接受 `vae_parallel_mode='tile'`。 + stages 1–3 在所有参与 rank 上处理完整 low-resolution feature volume，只有 stage 4 与 diffusion + stage 作为 overlapping tile task 分发，rank 0 按 serial reference 顺序 blend/merge。tile split 必须使 + 每个 rank 从 rank-0 generator state 出发，并保持与 serial tiled decode 相同的 noise-consumption + order；只有 active decoder 确认 collective decode 时非输出 rank 才参与。 +- 禁止：让不足 geometry 在 NATTEN/decode 中变成 HTTP 500；静默接受 height/width shard mode；按 rank + placement 改变固定 seed 的 tile noise，或用 ConvVAE 的 distributed-state 判定 DiffVAE collective。 +- 验收：覆盖 early geometry rejection 与 ConvVAE control、tile positive/non-tile negative modes、serial + RNG-order parity，以及 output/non-output rank 在 collective/non-collective decode 的分支。^[PR #6189] From b622b880854e4fe265ba217f288e8b3e98cb8318 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:11:09 +0800 Subject: [PATCH 336/465] knowledge(vllm-omni): sync 4245fd69 (PR #6694) --- adapters/vllm_omni/release_baseline.yaml | 16 +++--- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 24 +++++---- .../vllm-omni/models/mammoth-moda2/_index.md | 51 +++++++++++++++++++ .../vllm-omni/models/mammoth-moda2/rules.md | 45 ++++++++++++++++ 11 files changed, 126 insertions(+), 24 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/mammoth-moda2/_index.md create mode 100644 knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f2173eec0..5e917548b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 262ff146935553d444702b583874da461b50c079 + previous_audited_sha: 5e4daaf1970ef230c50d6fb2c7ef8ed4895545f8 audited_ref: main - audited_sha: 5e4daaf1970ef230c50d6fb2c7ef8ed4895545f8 + audited_sha: 4245fd69554536493bcda4e634aab9c84a6848fb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 81 - sha256: 9d5a685eae7ef099bb884557ca51b0e1b5f19c8555aa16bbd136c2081d95cd78 + count: 88 + sha256: 96d973728f1d67a959121c4aa0d8ba86cff32043ec6f90458f48d1d766acb8af diffusion: count: 65 sha256: 6107384ad499e668c1ed3690d740b7be816c900c504f244dce557ef51a257f2a pipelines: - count: 56 - sha256: 39e0a741462b11a2c96fa809be8ea88d502fbb4bc9068ab1a74ea6d7c6c6f1d6 + count: 58 + sha256: 45c68e3f444e8a1b30ba0e9d389078a5d1a17801bce8e28298b372204b6d4d29 deploy_yamls: - count: 85 - sha256: 205c0e5c20888852e8abb4b83d8fa81cc59066c7010f3db374f889f879ab6160 + count: 87 + sha256: 8aebd9640156fc63e40b783cf78dab2515798cc91505fe8c6e505f48097fdf55 # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ded7418b1..550cbca1b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5e4daaf1970ef230c50d6fb2c7ef8ed4895545f8`. The canonical machine baseline +4245fd69554536493bcda4e634aab9c84a6848fb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 26f16889d..5e7a961f9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5e4daaf1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4245fd69` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 97d7e1f33..cbe936d8d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5e4daaf1` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 4245fd69` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index dd174a5ac..602cf1417 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5e4daaf1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4245fd69` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b75f65b21..f262a8f0e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5e4daaf1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4245fd69` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 62e404003..ec6d306db 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5e4daaf1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4245fd69` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5eb55f1b5..454693490 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5e4daaf1` 验证存在 +- 源码校验：以上路径均已在 `main @ 4245fd69` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c4dfef99d..2ad4b4145 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5e4daaf1`（2026-08-27）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4245fd69`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -35,10 +35,10 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| -| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 81 个架构名 / 31 个模型族目录 | +| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 88 个架构名 / 32 个模型族目录 | | Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 40 个模型族目录 | -| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 56 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 85 份 | +| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 58 个 key | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 87 份 | PR #6089 新增 `nemotron_labs_voicechat_duplex.yaml`，因此只增加 deploy profile；它没有 增加 AR/diffusion registry architecture 或 `OMNI_PIPELINES` key。该 profile 是原生全双工 @@ -53,7 +53,7 @@ evaluation consumer, not a new model registration or deploy profile. ^[PR #6634] OMNI_PIPELINES 57→55，deploy 86→84。PR #6353 删除 AudioX 的 diffusion registry、 processor/extras/recipe/examples/tests，并删除 MammothModa2 Preview/Dev 共用的七个 AR architecture、两个 pipeline key、两个 deploy YAML 及其 tokenizer/config/AR/DiT/bridge。 -因此 active owner 与 catalog 不再声明这两个家族；diffusion text-to-audio 的 first-party +因此当时 active owner 与 catalog 不再声明这两个家族；diffusion text-to-audio 的 first-party 示例只剩 Stable Audio，Audex 的 TTA 仍是独立两阶段 autoregressive 路径。^[PR #6353] PR #6357 删除 `DreamIDOmniPipeline` 和 `MagiHumanPipeline`，以及相应的模型目录、MagiHuman @@ -62,6 +62,12 @@ first-party 支持，下面的 registry/owner 清单不将其列为 active。两 不得作为可运行支持或能力证据。RFC #5996 最初讨论的是 CI delist；本 PR 的合并代码和 PR body 明确扩大为完整 first-party 删除。^[PR #6357] ^[Issue #5996] +PR #6694 恢复 MammothModa2-Preview 和 MammothModa2-Dev：七个 AR architecture、 +`mammoth_moda2`（AR→DiT）与 `mammoth_moda2_ar`（AR-only）pipeline key，以及对应两份 +deploy YAML 回到 active inventory。Preview/Dev 的实现、prompt/extras 与 AR→DiT bridge 共享 +`mammoth_moda2` 家族；理解路径使用 AR-only topology，Dev 的 supported-model 表仅声明 AR-only +image understanding。详见 [MammothModa2 owner](mammoth-moda2/_index.md)。^[PR #6694] + PR #6619 删除独立的 `minicpmo_4_5_duplex.yaml`，将 duplex 配置并入全部 shipping MiniCPM-o profile，因此 deploy inventory 84→83；这是 profile 合并，不是 MiniCPM-o 能力删除。^[PR #6619] @@ -81,7 +87,7 @@ SANA-Video 2B 新增 native `SanaVideoPipeline`（T2V）和 aura_omni、audex、bagel、cosyvoice3、covo_audio、dots_tts、dynin_omni、fish_speech、 gepard、glm_image、glm_tts、higgs_audio_v2、higgs_audio_v3、hunyuan_image3、indextts2、 -mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、minimax_music3、moss_tts、 +mammoth_moda2、mimo_audio、ming_flash_omni、ming_tts、minicpmo_4_5、minimax_music3、moss_tts、 moss_tts_nano、nemotron_voicechat、omnivoice、personaplex、qwen2_5_omni、qwen3_omni、 qwen3_tts、step_audio2、voxcpm2、voxtral_tts @@ -102,7 +108,7 @@ dots_tts、dreamzero、dynin_omni、fish_qwen3_omni、gepard、glm_image、glm_t higgs_audio_v2、higgs_multimodal_qwen3、hunyuan_image3_ar、hunyuan_image3_dit、 hunyuan_image_3_moe、hunyuan_video_15、indextts2、indextts2_5、lance、mimo_audio、 ming_flash_omni、ming_flash_omni_image、ming_flash_omni_thinker_only、 -ming_flash_omni_tts、ming_tts、ming_tts_moe、minicpmo_4_5、minimax_music3、 +ming_flash_omni_tts、ming_tts、ming_tts_moe、mammoth_moda2、mammoth_moda2_ar、minicpmo_4_5、minimax_music3、 moss_tts_delay、moss_tts_local、moss_tts_nano、moss_tts_realtime、nemotron_labs_audex、 nemotron_labs_voicechat、nemotron_voicechat、omnivoice、personaplex、pi0、qwen2_5_omni、 qwen2_5_omni_thinker_only、qwen3_omni_moe（resolver）、qwen3_omni_moe_thinker_only、qwen3_tts、step_audio_2、 @@ -117,8 +123,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 262ff146 \ - --to 5e4daaf1 \ + --from 5e4daaf1 \ + --to 4245fd69 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/mammoth-moda2/_index.md b/knowledge/repos/vllm-omni/models/mammoth-moda2/_index.md new file mode 100644 index 000000000..f16b0fbc7 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/mammoth-moda2/_index.md @@ -0,0 +1,51 @@ +--- +title: "MammothModa2（Preview/Dev 的 AR→DiT 与 AR-only 拓扑）" +created: 2026-09-04 +updated: 2026-09-04 +type: index +tags: [vllm-omni, models, diffusion] +sources: ["PR #6694", vllm_omni/model_executor/models/mammoth_moda2/, vllm_omni/diffusion/models/mammoth_moda2/, vllm_omni/model_executor/stage_input_processors/mammoth_moda2.py, vllm_omni/deploy/mammoth_moda2.yaml, vllm_omni/deploy/mammoth_moda2_ar.yaml] +--- + +# MammothModa2 + +以下事实在 `main @ 4245fd69` 复核。 + +## 名称与范围 + +- Checkpoint：`bytedance-research/MammothModa2-Preview` 与 + `bytedance-research/MammothModa2-Dev`。PR #6694 恢复此前移除的家族，但不恢复 + AudioX。Preview/Dev 共用 `mammoth_moda2` 实现、tokenizer、typed transformers config、 + request extras 和 AR→DiT bridge。^[PR #6694] +- AR registry 有七个 architecture，均映射到 `model_executor/models/mammoth_moda2/`： + `MammothModa2Qwen2ForCausalLM`、`MammothModa2ARForConditionalGeneration`、 + `MammothModa2Qwen3ARForConditionalGeneration`、`MammothModa2Qwen3ForCausalLM`、 + `MammothModa2DiTPipeline`、`MammothModa2ForConditionalGeneration`、`Mammothmoda2Model`。 + DiT 实现在 `diffusion/models/mammoth_moda2/`；不要把它当成独立 diffusion-registry family。 +- `mammoth_moda2` 是两 stage 文生图：stage 0 `LLM_AR` 输出 latent，stage 1 + `LLM_GENERATION` 经 `stage_input_processors.mammoth_moda2.ar2dit` 生成 image，默认 + deploy 是 `mammoth_moda2.yaml`。`mammoth_moda2_ar` 是单 stage `LLM_AR` 文本输出， + 用于图像/视频理解和 summarization，默认 deploy 是 `mammoth_moda2_ar.yaml`。 + +## 维护入口 + +- prompt、image grid、EOL/visual-token constraints 或 text/image→text： + `model_extras/mammothmodal2_preview.py` 与 AR model consumer；共享 image task envelope + 见 [Model Executor](../../components/model-executor/rules-image-task-envelope.md)，模型专属 + 约束见 [MAMMO-1a](rules.md#mammo-1a-ar-sampling-必须按-task-隔离-text-与-visual-vocabulary)。 +- 改 AR→DiT payload：先沿 `ar2dit` 的 `full_hidden_states`、token ids、question/answer + boundary 和 dimensions 追到 DiT consumer；跨 stage 传输的一般合同见 + [bridge/batch rules](../../components/model-executor/rules-bridge-batch.md)，模型专属合同见同页 + `MAMMO-1b`。 +- 改 stage、default deploy 或 runtime topology：检查两个 `OMNI_PIPELINES` key、两个 + YAML 和 [Config 的 model registration entry](../../components/configuration/adding-a-model.md)。 + T2I deploy 的两个 stage 都默认 device `0`，这是 profile 默认值，不是通用硬件或性能承诺。 + +## 明确边界 + +- `text_guidance_scale`、`cfg_range`、`num_inference_steps` 经 model extras 的 + `extra_body` 进入 DiT；DiT 使用 kwargs 接口，因此不能假定 shared diffusion request + 字段会自动透传。 +- Preview/Dev 的支持范围以合并后的 upstream docs/registry 为准：Preview 有文生图与 + image understanding；Dev 在 supported-model 表只声明 AR-only image understanding。不要从 + PR 作者的单张 GPU 示例外推硬件、吞吐、质量或全部任务支持。^[PR #6694] diff --git a/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md b/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md new file mode 100644 index 000000000..7ca6d1c58 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md @@ -0,0 +1,45 @@ +--- +title: "MammothModa2 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, model-executor, diffusion] +sources: ["PR #6694", vllm_omni/model_extras/mammothmodal2_preview.py, vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py, vllm_omni/model_executor/stage_input_processors/mammoth_moda2.py, vllm_omni/diffusion/models/mammoth_moda2/pipeline_mammothmoda2_dit.py, tests/model_extras/test_model_extras.py, tests/e2e/offline_inference/test_mammoth_moda2_expansion.py] +confidence: high +--- + +# MammothModa2 规则 + +## MAMMO-1a — AR sampling 必须按 task 隔离 text 与 visual vocabulary + +- 触发：修改 Preview/Qwen2.5-VL 或 Dev/Qwen3-VL 的 T2I prompt、额外 generation vocabulary、 + EOL 约束、`generated_len` runtime metadata 或 `omni_task` routing。 +- 强制：T2I prompt 以 16-pixel patch 将输出尺寸映射为 AR grid，并携带 image-grid、EOL 与 + visual-token range；runner 必须为每个 request 注入实际 `generated_len`。T2I 的每行普通位置只 + 允许 visual-token range 且排除 EOL，在 `generated_len % (ar_width + 1) == ar_width` 时只允许 + EOL；chat/understanding task 必须屏蔽完整 extra generation vocabulary。Qwen2.5 与 Qwen3 consumer + 均维持这一分支，不能用 checkpoint 名或 pipeline 名代替 request task。 +- 禁止：让文本任务采样 image token；让 T2I 在行尾继续采样普通 visual token；用 batch-global + step 代替 per-request `generated_len`；或把兼容签名中的 `negative_prompt` 当成当前 T2I prompt + builder 已支持的显式 negative-conditioning 路径。 +- 验收：Preview/Dev config 的 base+visual vocab 边界可构造；T2I 覆盖行内 visual range 与行尾 + EOL-only，text control 覆盖 extra-vocab 全屏蔽；prompt builder 固定 task/grid/token metadata，且 + negative prompt 不被注入。^[PR #6694] + +## MAMMO-1b — AR→DiT bridge 必须保留 token/hidden-state 对齐 + +- 触发：修改 `mammoth_moda2` 两阶段 pipeline、`ar2dit`、AR multimodal output 或 DiT condition split。 +- 强制：stage 0 必须输出 `multimodal_output.latent`；bridge 将 prompt token ids 与 generation 的 + cumulative ids 拼接，但删除最后一个没有对应 hidden state 的 generated token，并断言 token 总数与 + hidden-state 第一维一致。跨 stage 传输前 hidden state 转为 contiguous FP32，以避开 numpy/BF16 + serializer 边界；同时传递 answer boundary、目标尺寸和 DiT sampling extras。DiT 从自己的 model + config 取得 generation-vocab 与 vision-placeholder ids，按 question/answer boundary 分离 text/image + conditions，并在没有生成 visual-token hidden state 时显式失败。 +- 尺寸优先级固定为 serving 写入的 `mm_processor_kwargs.target_h/target_w`、原 prompt + `additional_information`、最后才是 1024 default。`text_guidance_scale`、`cfg_range` 和 + `num_inference_steps` 通过 `extra_body → sampling.extra_args` 进入 DiT；legacy payload 仅作兼容 + fallback，不能假设标准 diffusion request 自动拥有这些 kwargs。 +- 验收：覆盖 single dict/list prompt normalization、缺 latent、token/hidden mismatch、尺寸优先级、 + empty visual condition、extra-body 参数透传和 AR-only text control。PR 作者报告的 54 个 CPU tests + 与单次 L20X Preview T2I PNG smoke 只证明该提交环境的路径可运行；golden-image E2E 仍因既有 issue + skip，不能据此声称质量、吞吐、Dev T2I 或跨硬件能力。^[PR #6694] From 58a18eba713be336e03a1f6d436012e63350c6db Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:16:02 +0800 Subject: [PATCH 337/465] knowledge(vllm-omni): sync 6ec5ab11 (PR #6692) --- adapters/vllm_omni/manifest.yaml | 8 ++++++ adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/host-weight-runtime/_index.md | 4 +-- .../components/host-weight-runtime/rules.md | 25 ++++++++++++++++++- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 12 files changed, 46 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index d32209732..cfbfff409 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -148,6 +148,14 @@ modules: test_paths: [tests/diffusion/test_inductor_divisibility_patch.py] wave: 1 risk: high + # Filesystem-backed immutable host artifacts have process-wide lock, lease, + # durability, and fallback contracts; keep this owner explicit rather than + # letting generic runtime scopes silently omit it. + host_weight_runtime: + local_paths: [vllm_omni/host_weight_runtime/] + test_paths: [tests/host_weight_runtime/] + wave: 1 + risk: high sampling: local_paths: [vllm_omni/sample/] wave: 1 diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 5e917548b..2eaeda286 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5e4daaf1970ef230c50d6fb2c7ef8ed4895545f8 + previous_audited_sha: 4245fd69554536493bcda4e634aab9c84a6848fb audited_ref: main - audited_sha: 4245fd69554536493bcda4e634aab9c84a6848fb + audited_sha: 6ec5ab11bdd618b46e3b5ebd9b8d52e336a2abd5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 550cbca1b..2328613e5 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4245fd69554536493bcda4e634aab9c84a6848fb`. The canonical machine baseline +6ec5ab11bdd618b46e3b5ebd9b8d52e336a2abd5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5e7a961f9..386e4aef2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4245fd69` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6ec5ab11` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index cbe936d8d..9c4e521cd 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4245fd69` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 6ec5ab11` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 602cf1417..c5bff332a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4245fd69` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6ec5ab11` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md index a073ac25d..3a9cfd56a 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/_index.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: index tags: [vllm-omni, components] -sources: ["PR #6419", "PR #6486", "PR #6591", vllm_omni/host_weight_runtime/] +sources: ["PR #6419", "PR #6486", "PR #6591", "PR #6692", vllm_omni/host_weight_runtime/] confidence: high --- @@ -24,4 +24,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| typed failure、lease/store concurrency、single-take carrier、restore transaction | [rules](rules.md) | +| typed failure、lease/store concurrency、single-take carrier、restore transaction，以及 hardened artifact 的 cleanup/quarantine/publication、deny-marker 或 durability I/O failure | [rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md index 63f01596e..4be763028 100644 --- a/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md +++ b/knowledge/repos/vllm-omni/components/host-weight-runtime/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, components] -sources: ["PR #6419", "PR #6427", "PR #6445", "PR #6486", "PR #6591", vllm_omni/host_weight_runtime/] +sources: ["PR #6419", "PR #6427", "PR #6445", "PR #6486", "PR #6591", "PR #6692", vllm_omni/host_weight_runtime/] confidence: high --- @@ -62,6 +62,29 @@ confidence: high - 验收：覆盖 ordered write-time digest、unordered parallel readback、payload `fsync` failure、digest mismatch 与 publication barrier，断言失败不产生可命中的 `READY` artifact。^[PR #6591] +## HWR-1e — filesystem lifecycle transition 必须 fail-closed + +- 触发：修改 filesystem store 的 publication、deny、cleanup、quarantine、rename、chmod 或 directory + `fsync` 路径。 +- 强制：每个 artifact move 均在 exclusive artifact lock 下先持久标记打开 inode 为 invalid，再临时 + 加 owner-write、atomic rename、re-harden 并同步两个 parent，最后清除 marker；新 staging artifact + 将 marker 随 rename 带入，只有 harden + durability barrier 后才成为 ready。合作 reader 在 open 和 + 返回 lease 前检查 marker；中断 cleanup 的 `.cleanup.*` tombstone 由后续 exact-key cleanup/build + 重试，quarantine 中带 marker 的过渡目录先 re-harden。deny marker 发布失败时，同步标记并按 inode + 比较后 containment，绝不隔着 lock handoff quarantine 新的 replacement。 +- 强制：结构/manifest/checksum 语义错误仍是 validation invalid；暂态 read/hash/deny/lock/durability + I/O 是 retryable storage failure，保留未知 competing entry。rename 后、authority commit 前的任一 + 失败必须 containment；marker 与 quarantine 都不能持久化时返回 retryable、`outcome_uncertain` + failure，不能报告可被后续 HIT 反驳的 non-authoritative publication failure。authority commit 后的 + lock-release 类错误只记录，不得把 ready artifact 改报失败。 +- 禁止：将 hardened artifact 直接 rename（overlayfs/tmpfs 可拒绝）；把暂态 I/O 伪装为 semantic + corruption；在 containment 不确定时允许 weaker lookup lease；在已提交 ready entry 上返回相反的 + publication failure。 +- 验收：在 overlayfs/tmpfs 覆盖 hardened cleanup/quarantine/rebuild；故障注入覆盖 pre/post-rename + crash、chmod/rename/parent-sync/deny-write failure、marker+quarantine double failure、weaker lookup + race、competing manifest I/O 与 post-authority release，并断言 typed retryability、inode-safe + containment、tombstone repair 和无 contradictory HIT。^[PR #6692] + ## HWR-2a — restore 是 validation-only plan 加一次性 commit - 触发：manifest/schema、producer/restorer identity 或模型 hydration 变化。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f262a8f0e..d38b3e6f8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4245fd69` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6ec5ab11` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ec6d306db..55bd102b7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4245fd69` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6ec5ab11` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 454693490..be863a952 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4245fd69` 验证存在 +- 源码校验：以上路径均已在 `main @ 6ec5ab11` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2ad4b4145..6809c4144 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4245fd69`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6ec5ab11`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -123,8 +123,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 5e4daaf1 \ - --to 4245fd69 \ + --from 4245fd69 \ + --to 6ec5ab11 \ --repo \ --mode report-only ``` From c522a6976da9773285a4f2e7ce9dc79d5b66c43f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:22:50 +0800 Subject: [PATCH 338/465] knowledge(vllm-omni): sync f8f8aa8e (PR #6609) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 3 +- .../components/serving/rules-app-assembly.md | 28 +++++++++++++++++++ .../vllm-omni/components/serving/rules.md | 4 +-- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 11 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2eaeda286..fc25cd272 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4245fd69554536493bcda4e634aab9c84a6848fb + previous_audited_sha: 6ec5ab11bdd618b46e3b5ebd9b8d52e336a2abd5 audited_ref: main - audited_sha: 6ec5ab11bdd618b46e3b5ebd9b8d52e336a2abd5 + audited_sha: f8f8aa8eb760d79499e43fea712a181b83dbbf5b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2328613e5..4bcb43ca9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6ec5ab11bdd618b46e3b5ebd9b8d52e336a2abd5`. The canonical machine baseline +f8f8aa8eb760d79499e43fea712a181b83dbbf5b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 386e4aef2..d56566969 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6ec5ab11` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f8f8aa8e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9c4e521cd..5581adf04 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6ec5ab11` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ f8f8aa8e` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c5bff332a..d0614dbf0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6ec5ab11` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f8f8aa8e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index d38b3e6f8..276f7d9f2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6ec5ab11` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f8f8aa8e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 55bd102b7..633b6727a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6ec5ab11` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f8f8aa8e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index be863a952..be01493c5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6ec5ab11` 验证存在 +- 源码校验：以上路径均已在 `main @ f8f8aa8e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -33,5 +33,6 @@ sources: [] | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | | engine startup/shutdown、stage lifecycle、full-duplex、CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | +| assembled FastAPI route 覆盖、method/path ownership 或 request-body forwarding | [app assembly 规则](rules-app-assembly.md) | | stage config→EngineArgs projection、explicit `devices`、TP/local-DP/PP、replica layout 或 worker 创建前的 layout guard | [stage 启动与设备布局规则](rules-stage-startup.md) | | 公开请求字段的校验、来源冲突、alias/extras 归一与 consumer view | [请求输入合同](rules-request-input.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md b/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md new file mode 100644 index 000000000..084985c17 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md @@ -0,0 +1,28 @@ +--- +title: "Serving app assembly 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #6609", vllm_omni/entrypoints/openai/api_server.py, tests/entrypoints/openai/test_profiler_endpoints.py, tests/entrypoints/openai_api/test_api_server_guards.py] +confidence: high +--- + +# Serving app assembly 规则 + +## SERV-5s — 覆盖 upstream HTTP route 前必须先移除同 method/path 的旧 route + +- 触发：assembled app 用 Omni handler 覆盖 upstream HTTP endpoint，尤其是附带可选 request body + 或 stage scope 的 conditional router。 +- 强制：在 `include_router()` 前，从 assembled app 移除相同 path 与 HTTP method 的 upstream route； + 只在该 feature 启用的同一条件分支执行替换。FastAPI/Starlette 不会因后注册同 path/method 而替换 + 旧 route，dispatch 会命中先注册者。保留无关 upstream routes。 +- 禁止：只检查 router 自身存在，或只断言 HTTP 200；这不能证明 assembled app 没有重复 route， + 也不能证明 request body 到达 Omni handler。 +- 验收：assembly test 为 upstream 和 Omni 注册相同 method/path，断言仅剩一个 route 且 endpoint + 属于 Omni；公开 endpoint test 发送 `stages` 并断言 engine client 收到原值。当前 + `/start_profile` 与 `/stop_profile` 是此合同的实例。^[PR #6609] +- 已知边界：同一 assembled app 的 upstream `GET /health` 仍先于 Omni health route 注册，当前 + first-match dispatch 会遮蔽 Omni 的 diffusion fallback/JSON handler；测试 fake upstream 也未注册 + 该 sibling route。该 review thread 在本提交获批时明确 defer，不能据 profiler 修复声称所有重复 + route 已清零，后续修复还需核对 render-only `engine_client is None` 时 200/503 的语义差异。^[PR #6609] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index ba46c068c..58953c5b8 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -22,7 +22,7 @@ confidence: high | `extra_body`、flattened/nested/canonical/legacy 输入、alias、`negative_prompt`、diffusion request extras | `request-contract`：`SERV-4a`–`4h` | `vllm_omni/entrypoints/openai/diffusion_request_utils.py::{normalize_diffusion_request_args,apply_normalized_diffusion_request_extra_args}` → `serving_chat.py::{OmniOpenAIServingChat._preprocess_chat,OmniOpenAIServingChat.generate_diffusion_images}` | | batched chat、fan-out/fan-in、sub-request ID、choice collapse、whole-batch error | `batch-chat-contract`：`SERV-4i`–`4k` | `api_server.py::create_batch_chat_completion` → `batch_serving.py::OmniOpenAIServingChatBatch` → ordinary chat completion children | | `chat_template_kwargs`、raw HTTP/SDK `extra_body`、text/audio modalities、choices、空音频 | `chat-multimodal-contract`：`SERV-4c` + 命中模型规则 | upstream `ChatCompletionRequest` → `serving_chat.py::{OmniOpenAIServingChat._preprocess_chat,OmniOpenAIServingChat.chat_completion_full_generator,OmniOpenAIServingChat._create_text_choice,OmniOpenAIServingChat._create_audio_choice}` | -| endpoint restriction、route/app-state guard、capability、公开 400 | `endpoint-capability`：`SERV-4c`, `SERV-4d`, `SERV-5d` | endpoint policy → `api_server.py::build_app` assembled app → public handler | +| endpoint restriction、route/app-state guard、capability、公开 400 | `endpoint-capability`：`SERV-4c`, `SERV-4d`, `SERV-5d`, [SERV-5s](rules-app-assembly.md#serv-5s-覆盖-upstream-http-route-前必须先移除同-methodpath-的旧-route) | endpoint policy → `api_server.py::build_app` assembled app → public handler | | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup acknowledgment | `engine-lifecycle`：`SERV-5a`, `SERV-5b`, `SERV-5m` | `entrypoints/async_omni.py::{AsyncOmni.pause_generation,AsyncOmni.resume_generation,AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni._abort}` → `engine/{stage_pool,orchestrator,async_omni_engine}.py` → AR EngineCore helper / diffusion worker RPC | | stage/replica death、remote membership register/watch/detach、`EngineDeadError`、`StageUnavailableError`、health/readiness、eviction | `engine-lifecycle`：[SERV-5f](rules-fault-isolation.md#serv-5f-stage-death-按-replica-隔离request-fatal-不等于-process-fatal)、[SERV-5n](rules-engine-lifecycle.md#serv-5n-remote-replica-attachdetach-必须按本次-membership-generation-判定) | `engine/membership_controller.py` register/watcher generation → `engine/stage_pool.py` live set → `engine/orchestrator.py` poll/dispatch/cleanup → `entrypoints/{omni_base,async_omni}.py` error transport | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | @@ -44,7 +44,7 @@ confidence: high | `metrics-lifecycle` | metrics、gauge、replica、collector、image/diffusion measurement boundary | `SERV-2a`, `SERV-2b`, `SERV-2e` | | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | -| `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 本页 | +| `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 见 engine lifecycle；`SERV-5s` 见 [app assembly](rules-app-assembly.md) | | `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、streaming raw terminal、event-driven orchestration、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5q`（见 [engine 生命周期规则](rules-engine-lifecycle.md)），`SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds | `SERV-4a`–`4h`, `SERV-4l`–`4p`，全部见 [请求输入合同](rules-request-input.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6809c4144..8792f60ab 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6ec5ab11`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f8f8aa8e`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -123,8 +123,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 4245fd69 \ - --to 6ec5ab11 \ + --from 6ec5ab11 \ + --to f8f8aa8e \ --repo \ --mode report-only ``` From 9169829ac370bd5a1d8d4181972c610d3e75ac55 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:32:27 +0800 Subject: [PATCH 339/465] knowledge(vllm-omni): sync ab8d45d3 (PR #6587) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/minicpm-o-4-5/_index.md | 7 +++-- .../models/minicpm-o-4-5/rules-cuda-graphs.md | 28 +++++++++++++++++ .../vllm-omni/models/minicpm-o-4-5/rules.md | 31 ++----------------- 12 files changed, 47 insertions(+), 43 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-cuda-graphs.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index fc25cd272..36078cd11 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6ec5ab11bdd618b46e3b5ebd9b8d52e336a2abd5 + previous_audited_sha: f8f8aa8eb760d79499e43fea712a181b83dbbf5b audited_ref: main - audited_sha: f8f8aa8eb760d79499e43fea712a181b83dbbf5b + audited_sha: ab8d45d3cc4e6bf30116dd6a5a62fd193ec51cc1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4bcb43ca9..54281525c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f8f8aa8eb760d79499e43fea712a181b83dbbf5b`. The canonical machine baseline +ab8d45d3cc4e6bf30116dd6a5a62fd193ec51cc1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d56566969..b81b809c7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f8f8aa8e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ab8d45d3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5581adf04..47ed28e8c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f8f8aa8e` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ab8d45d3` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d0614dbf0..8881a3d46 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f8f8aa8e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ab8d45d3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 276f7d9f2..86f0ed421 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f8f8aa8e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ab8d45d3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 633b6727a..87479a225 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f8f8aa8e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ab8d45d3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index be01493c5..ec722c0e3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f8f8aa8e` 验证存在 +- 源码校验：以上路径均已在 `main @ ab8d45d3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8792f60ab..76ce51330 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f8f8aa8e`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ab8d45d3`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -123,8 +123,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 6ec5ab11 \ - --to f8f8aa8e \ + --from f8f8aa8e \ + --to ab8d45d3 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 7c1614a4c..f33e6d8c5 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6678", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082"] +sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6678", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082", "PR #6587"] confidence: high --- @@ -33,7 +33,8 @@ Step-Audio2 的 token2wav。engine cache/profile 与 fallback 门禁见 MCPMO-1c 四份 bundled deploy 在 CUDA 上另默认开启 HiFT graph：只 capture pre-iSTFT 子图，按 connector chunk/cache shape 预捕并限量 lazy capture，且不服从 stage `enforce_eager`；非 CUDA 回 eager。 -shape、显存、并发与部分-graph 性能证据边界见 MCPMO-1d。 +HiFT/CFM 的 shape、显存、并发、整代退休与部分-graph 性能证据边界见 +[Code2Wav CUDA graph 规则](rules-cuda-graphs.md)。 Thinker 的 Whisper/APM audio encoder 仍构造 dense `[B,1,T,T]` mask；chunk mask 已用 broadcasted query/key index 代替逐 row Python fill，但不改变 chunk/left-context/lookahead @@ -56,7 +57,7 @@ state、batch cap 与 NPU residual limits 见 [Code2Wav 并发批处理规则](r ## 什么时候查这里 -- 审查 MiniCPM-o 4.5 registry、remote-code gate、TTS dependency、Code2Wav TensorRT/HiFT graph、 +- 审查 MiniCPM-o 4.5 registry、remote-code gate、TTS dependency、Code2Wav TensorRT/HiFT/CFM graph、 batch/stage handoff 或 native duplex session。 - 问题位于共享 bridge/batching 时转到 [Model Executor rules](../../components/model-executor/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-cuda-graphs.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-cuda-graphs.md new file mode 100644 index 000000000..816847257 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-cuda-graphs.md @@ -0,0 +1,28 @@ +--- +title: "MiniCPM-o 4.5 Code2Wav CUDA graph 规则" +created: 2026-09-04 +updated: 2026-09-04 +type: rule +tags: [vllm-omni, models, model-executor] +sources: [vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, "PR #5869", "PR #6082", "PR #6587"] +confidence: high +--- + +# MiniCPM-o 4.5 Code2Wav CUDA graph 规则 + +## MCPMO-1d — HiFT graph 只捕获稳定 pre-iSTFT 子图并限界 shape cache + +- 触发：`enable_hift_graph`、capture batch/chunk 配置、HiFT inference 分段或 streaming cache shape。 +- 强制：只在 parameter device 为 CUDA 时启用；非 CUDA 回 eager。capture bucket 由 codec chunk、left context、Flow lookahead、token→mel ratio 与 mel/source cache 推导；启动预捕 uncached/cached shape，未知 final shape 最多 lazy capture 8 个，超限或无可容纳 batch 时回 eager。active stream capture 中禁止嵌套 capture/replay并回 eager；静态输入先清零再复制真实 batch，输出 slice 后 clone。 +- 强制：graph 只覆盖 `_inference_pre_istft`；`torch.istft` 的 NOLA CPU-GPU sync 与 waveform clamp 留在 eager `_finalize_decode`。HiFT window/harmonic IDs 必须是随 module device 移动的 non-persistent buffers。开关独立于 stage `enforce_eager`；四份 bundled MiniCPM-o deploy 默认开启。 +- 禁止：无 connector 配置、非正 chunk、负 left context、source/mel cache 不整除时静默 capture；将未来 `initial_codec_chunk_frames` 当已预捕（当前会 lazy capture）；把 lazy graph limit 说成总显存上界，因为启动 graphs、global pool、static tensors 与其他 graph owners 仍驻留。active-capture 分支虽绕过 nested replay，却调用包含 NOLA sync 的完整 eager decode；未做真实 outer-graph 测试前，不能把日志中的 “fallback” 当成可捕获保证。custom `capture_batch_sizes` 也尚未校验正数/去重。 +- 验收：CUDA test 比较 cached/uncached graph 与 eager，CPU/mock 覆盖 lazy capture/limit fallback，deploy test 固定默认开关；另测非 CUDA disable、invalid config、unsupported batch、nested capture 和 variable final chunk。static input/output 与 lazy graph map 为无锁可变状态；并发 replay/capture 需先证明被上层串行化或增加互斥与重叠调用测试。PR 的单 A800 profile 约 30→5 ms/chunk、250→43 ms/request，只绑定部分 graph、单 prompt/commit `8fa28d88`，无 repeats/端到端质量，不能泛化为稳定 speedup。^[PR #5869] + +## MCPMO-1e — CFM DiT CUDA Graph 必须按 shape 整代退休并保持 eager parity + +- 触发：修改 MiniCPM-o 4.5 Code2Wav CFM DiT estimator 的 CUDA Graph 开关、`blocks_forward_chunk` graph target、按 shape 捕获/重放、cache retirement 或 eager fallback。 +- 强制：仅在 CUDA 且显式启用时捕获 `estimator.blocks_forward_chunk`；`t_embedder`、`torch.cat`、`expand`、buffer 准备和 10-step Euler loop 保持 eager。graph key 必须包含所有输入的 shape、dtype 和 device；`cnn_cache=None` 与 `att_cache=None` 转为 graph 可接受的零值时，必须保持与 eager `[None] * depth` 的首 chunk 数值语义，重放结果和 cache 输出需 clone。 +- 强制：`cfm_max_graphs` 是一代的有界容量。满载遇到新 shape 时，先同步，再只对本 wrapper 拥有的全部 `CUDAGraph` 调用 `reset()` 并清空 cache，随后捕获新一代的首图；绝不可逐 entry/LRU eviction。不得进行 process-wide cuBLAS workspace 清理或 `empty_cache()`：HiFT 等共享 global graph pool 的存活图不属于 CFM 的退休范围。`max_graphs <= 0` 必须禁用捕获并 eager。 +- 强制：warmup 或 capture 发生异常时，在该 wrapper 的剩余生命周期内保持禁用并退休其自有 generation；不能在可能脏的 capture stream/allocator 状态继续捕获。无法由 key 重建的 dtype 仅标记该 shape eager，不得禁用其他可捕获 shape；capture admission 不得依赖 device free-memory。维护固定字段的 calls/hits/captures/flushes/eager 累计 telemetry。 +- 禁止：捕获整个 `_estimator_step` 或手写替代 graph target；使用无界 shape cache、逐图 eviction、跨 owner 的 process-wide cleanup，或在 nested capture 中重放；捕获失败后伪装成功；把 CFM graph 的配置、性能或 parity 结论外推到 HiFT、encoder、TRT 或其他模型。 +- 验收：CUDA 测试覆盖 uncached/cached shape 的 graph/eager 数值 parity、`None` cache parity、lazy capture、整代 flush 后 HiFT replay parity、capture/warmup failure 后不再 capture、unsupported dtype shape-local eager 与 `max_graphs <= 0` eager；CPU/mock 覆盖 active capture/non-CUDA、free-memory 非 gating、telemetry 和 inert non-CUDA memory reporting，并确认 deploy 配置值实际到达 estimator。^[PR #6082] ^[PR #6587] diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index bfa1f0e05..0c932e8f3 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-04 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6619"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619"] confidence: high --- @@ -22,7 +22,8 @@ confidence: high | tokenizer/processor、`trust_remote_code` | MCPMO-1a | pipeline/config factory → `model_executor/models/minicpmo_4_5/` loader | | TTS extra、backend 初始化、空音频 | MCPMO-1b | `minicpmo_4_5_omni_tts.py::MiniCPMO45OmniTTSForConditionalGeneration`、`minicpmo_4_5_token2wav.py::MiniCPMO45Token2wav` | | Code2Wav TensorRT、DiT/Campplus、engine cache/profile | MCPMO-1c | `minicpmo_4_5_code2wav.py::MiniCPMO45Code2Wav` → `batched_token2wav.py::BatchedToken2Wav._estimator_step`；共享实现 `step_audio2/step_audio2_dit_trt.py` | -| HiFT CUDA Graph、chunk bucket、lazy capture、iSTFT | MCPMO-1d | `cuda_graph_wrapper.py::HiFTGraphWrapper` → `BatchedToken2Wav._hift_inference` → shared `HiFTGenerator` | +| HiFT CUDA Graph、chunk bucket、lazy capture、iSTFT | [`MCPMO-1d`](rules-cuda-graphs.md#mcpmo-1d-hift-graph-只捕获稳定-pre-istft-子图并限界-shape-cache) | `cuda_graph_wrapper.py::HiFTGraphWrapper` → `BatchedToken2Wav._hift_inference` → shared `HiFTGenerator` | +| CFM DiT CUDA Graph、shape cache、generation retirement、eager fallback | [`MCPMO-1e`](rules-cuda-graphs.md#mcpmo-1e-cfm-dit-cuda-graph-必须按-shape-整代退休并保持-eager-parity) | `cuda_graph_wrapper.py::CFMGraphWrapper` → `BatchedToken2Wav._estimator_step` | | batch、`runtime_info`、stage handoff | MCPMO-3a/3b | `stage_input_processors/minicpmo_4_5_omni.py` → `minicpmo_4_5_omni.py::MiniCPMO45OmniForConditionalGeneration` → TTS/code2wav | | Talker codec sampling、repetition penalty、request RNG/compaction | MCPMO-3c | `minicpmo_4_5_omni_tts.py::{make_omni_output,_sample_audio_codes,_apply_batched_repetition_penalty}` → `test_talker_batching.py` | | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | @@ -69,32 +70,6 @@ confidence: high 复用、不同 SM 不共享 plan、CFG 后 batch 越界以及 chunk/cache 越界。PR 未新增自动化测试， 因而这些仍是后续改动必须补证的验证缺口。 ^[PR #5638] -## MCPMO-1d — HiFT graph 只捕获稳定 pre-iSTFT 子图并限界 shape cache - -- 触发：`enable_hift_graph`、capture batch/chunk 配置、HiFT inference 分段或 streaming cache shape。 -- 强制：只在 parameter device 为 CUDA 时启用；非 CUDA 回 eager。capture bucket 由 codec chunk、 - left context、Flow lookahead、token→mel ratio 与 mel/source cache 推导；启动预捕 uncached/cached - shape，未知 final shape 最多 lazy capture 8 个，超限或无可容纳 batch 时回 eager。active stream - capture 中禁止嵌套 capture/replay并回 eager；静态输入先清零再复制真实 batch，输出 slice 后 clone。 -- 强制：graph 只覆盖 `_inference_pre_istft`；`torch.istft` 的 NOLA CPU-GPU sync 与 waveform clamp - 留在 eager `_finalize_decode`。HiFT window/harmonic IDs 必须是随 module device 移动的 non-persistent - buffers。开关独立于 stage `enforce_eager`；四份 bundled MiniCPM-o deploy 默认开启。 -- 禁止：无 connector 配置、非正 chunk、负 left context、source/mel cache 不整除时静默 capture； - 将未来 `initial_codec_chunk_frames` 当已预捕（当前会 lazy capture）；把 lazy graph limit 说成总显存 - 上界，因为启动 graphs、global pool、static tensors 与其他 graph owners 仍驻留。active-capture - 分支虽绕过 nested replay，却调用包含 NOLA sync 的完整 eager decode；未做真实 outer-graph 测试前， - 不能把日志中的 “fallback” 当成可捕获保证。custom `capture_batch_sizes` 也尚未校验正数/去重。 -- 验收：CUDA test 比较 cached/uncached graph 与 eager，CPU/mock 覆盖 lazy capture/limit fallback， - deploy test 固定默认开关；另测非 CUDA disable、invalid config、unsupported batch、nested capture 和 - variable final chunk。static input/output 与 lazy graph map 为无锁可变状态；并发 replay/capture 需先 - 证明被上层串行化或增加互斥与重叠调用测试。PR 的单 A800 profile 约 30→5 ms/chunk、 - 250→43 ms/request，只绑定部分 graph、 - 单 prompt/commit `8fa28d88`，无 repeats/端到端质量，不能泛化为稳定 speedup。^[PR #5869] - -## MCPMO-1e — CFM DiT CUDA Graph 必须按 shape 捕获并保持 eager parity - -- 触发：修改 MiniCPM-o 4.5 Code2Wav CFM DiT estimator 的 CUDA Graph 开关、`blocks_forward_chunk` graph target、按 shape 捕获/重放、cache eviction 或 eager fallback。\n- 强制：仅在 CUDA 且显式启用时捕获 `estimator.blocks_forward_chunk`；`t_embedder`、`torch.cat`、`expand`、buffer 准备和 10-step Euler loop 保持 eager。graph key 必须包含所有输入的 shape、dtype 和 device，并使用有界 `cfm_max_graphs` LRU；`cnn_cache=None` 与 `att_cache=None` 转为 graph 可接受的零值时，必须保持与 eager `[None] * depth` 的首 chunk 数值语义，重放结果和 cache 输出需 clone。\n- 禁止：捕获整个 `_estimator_step` 或手写替代 graph target；使用无界 shape cache；在 nested capture 中重放；捕获失败后伪装成功；把 CFM graph 的配置、性能或 parity 结论外推到 HiFT、encoder、TRT 或其他模型。\n- 验收：CUDA 测试覆盖 uncached/cached shape 的 graph/eager 数值 parity、`None` cache parity、lazy capture、capture failure eager fallback 和 LRU eviction；同时覆盖 active capture/non-CUDA fallback，并确认 deploy 配置值实际到达 estimator。\n^[PR #6082] - ## MCPMO-1f — Ascend Code2Wav NPUGraph 只捕获精确形状的 CFM DiT - 触发：MiniCPM-o 4.5 在 Ascend 上修改 Code2Wav、CFM DiT estimator、NPUGraph 开关、图缓存或 bundled deploy profile。 From 8fac053f91191d9baf3934ccd28c2aeab05e0beb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:37:16 +0800 Subject: [PATCH 340/465] knowledge(vllm-omni): sync ed837e35 (PR #6688) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minimax-h3/rules-media.md | 8 +++++++- 10 files changed, 19 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 36078cd11..42cc996d2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f8f8aa8eb760d79499e43fea712a181b83dbbf5b + previous_audited_sha: ab8d45d3cc4e6bf30116dd6a5a62fd193ec51cc1 audited_ref: main - audited_sha: ab8d45d3cc4e6bf30116dd6a5a62fd193ec51cc1 + audited_sha: ed837e350e18315ef5e2930b1bf33458036a3afc # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 54281525c..ee0a2844a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ab8d45d3cc4e6bf30116dd6a5a62fd193ec51cc1`. The canonical machine baseline +ed837e350e18315ef5e2930b1bf33458036a3afc`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b81b809c7..add0fdfd2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ab8d45d3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ed837e35` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 47ed28e8c..9186153f4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ab8d45d3` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ ed837e35` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8881a3d46..a1f7dbad4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ab8d45d3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ed837e35` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 86f0ed421..e408314fa 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ab8d45d3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ed837e35` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 87479a225..3b9b8b715 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ab8d45d3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ed837e35` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ec722c0e3..6dda046ae 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ab8d45d3` 验证存在 +- 源码校验：以上路径均已在 `main @ ed837e35` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 76ce51330..53fb3701c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ab8d45d3`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ed837e35`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -123,8 +123,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from f8f8aa8e \ - --to ab8d45d3 \ + --from ab8d45d3 \ + --to ed837e35 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md index 101e531fe..4ecb9cf48 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5752", "PR #5829", "PR #5978", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064"] +sources: ["PR #5752", "PR #5829", "PR #5978", "PR #6555", "PR #6688", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064"] confidence: high --- @@ -80,6 +80,12 @@ confidence: high nightly 声明 4×H100；PR 的 I2VA 0.975190/35.981286、Ref2VA 0.980577/ 42.650204 来自 4×B300 外部运行，不是 checked-in 结果。T2VA 因严格 gate 未达标被删除，仅表示无 retained pixel-level gate，不表示功能不支持。^[PR #5978] + 两个 official-reference request 都是 fixed-shape：测试 shared server args 必须以 + `--no-diffusion-compile-dynamic` 保持 regional compile static，避免 cold dynamic + Inductor codegen 在 CI 节点间造成 precision drift；cache 状态是此前 bisect 缺失的变量。 + 这仍覆盖 compiled production path，但仅是这两个 retained accuracy case 的 test + configuration，不能推广为 production 默认。它不改 SSIM/PSNR gate、attention backend、 + cuDNN settings 或 model implementation。^[PR #6688] ## MMH3-2i — 参考视频解码必须按需求索引并闭合 RGB 字节合同 From 05fb536a98c89ec6d0f5c0a599251134acb90a9c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:48:40 +0800 Subject: [PATCH 341/465] knowledge(vllm-omni): sync b6a4baa6 (PR #5885) --- adapters/vllm_omni/release_baseline.yaml | 16 +++++------ doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../configuration/rules-topology-profiles.md | 10 +++++-- .../components/configuration/rules.md | 4 +-- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-output-lifecycle.md | 12 ++++----- .../vllm-omni/components/diffusion/rules.md | 1 + .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-request-input.md | 11 ++++++-- .../vllm-omni/components/serving/rules.md | 3 ++- knowledge/repos/vllm-omni/models/catalog.md | 18 ++++++++----- .../vllm-omni/models/minimax-h3/_index.md | 12 ++++++++- .../models/minimax-h3/rules-media.md | 27 ++++++++++++++++++- .../vllm-omni/models/minimax-h3/rules.md | 1 + 18 files changed, 92 insertions(+), 37 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 42cc996d2..84218a1e6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,25 +2,25 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ab8d45d3cc4e6bf30116dd6a5a62fd193ec51cc1 + previous_audited_sha: ed837e350e18315ef5e2930b1bf33458036a3afc audited_ref: main - audited_sha: ed837e350e18315ef5e2930b1bf33458036a3afc + audited_sha: b6a4baa659238e5e13a2f295430820a3ae36dde5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. inventories: autoregressive: - count: 88 - sha256: 96d973728f1d67a959121c4aa0d8ba86cff32043ec6f90458f48d1d766acb8af + count: 89 + sha256: cd6b56fe44e88ca35e4d2c961a00cf7a257695ca193e8279d6562eb72ca58fc4 diffusion: count: 65 sha256: 6107384ad499e668c1ed3690d740b7be816c900c504f244dce557ef51a257f2a pipelines: - count: 58 - sha256: 45c68e3f444e8a1b30ba0e9d389078a5d1a17801bce8e28298b372204b6d4d29 + count: 59 + sha256: 4f012fb016221c2115520b16577bb69be03e84987122729bea696c14e5bca42f deploy_yamls: - count: 87 - sha256: 8aebd9640156fc63e40b783cf78dab2515798cc91505fe8c6e505f48097fdf55 + count: 89 + sha256: 7895ed1c963b0d2a4afedda6a45c422288973a50c73a1255d271a8beefe0a39f # Structural review owners for changed upstream paths. This is deliberately # separate from manifest.modules: changing that runtime map alters rebase fan-out. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ee0a2844a..7efdb5f46 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ed837e350e18315ef5e2930b1bf33458036a3afc`. The canonical machine baseline +b6a4baa659238e5e13a2f295430820a3ae36dde5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index add0fdfd2..fd7ab0dbc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ed837e35` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b6a4baa6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md index e5717cc81..f513748d5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md @@ -4,13 +4,13 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, config] -sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #6082", vllm_omni/deploy/, "PR #6186"] +sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/deploy/, "PR #6186"] confidence: high --- # topology 与部署 profile 合同 -`CONF-5a`–`CONF-5g`：冻结 topology 的单一来源、可选辅助 stage 注入、缺少 model_type 的多 stage checkpoint 路由，以及模型专用 deploy profile 的一致传递。触发条件与其余审查组见 [Configuration 规则](rules.md) 的 Direct 代码快速入口。 +`CONF-5a`–`CONF-5i`：冻结 topology 的单一来源、可选辅助 stage 注入、缺少 model_type 的多 stage checkpoint 路由，以及模型专用 deploy profile 的一致传递。触发条件与其余审查组见 [Configuration 规则](rules.md) 的 Direct 代码快速入口。 ## CONF-5a — 冻结 topology 只保留一份，部署开关决定 wiring @@ -77,3 +77,9 @@ confidence: high - 禁止：只依赖路径 basename、文件存在或 `trust_remote_code` 推断 pipeline；让 root architecture 覆盖 stage architecture；让单卡与多卡 profile 复制不同 topology 或行为合同。 - 验收：用 bare root config、无显式 deploy 的 startup 和 structured/legacy 路径断言 pipeline、默认 deploy、stage 子目录、architecture 与 sampling-extra key 均进入最终逐 stage config；单卡与双卡配置除设备 placement 外保持一致。 ^[PR #6186] +## CONF-5i — pipeline 专属 alias 与 hook 必须在通用配置层声明、归一并消费 + +- 触发：pipeline 需要把全局 CLI 字段改写到特定 stage，或声明 stage-0 prompt transform、按 task 解析 checkpoint 的 hook、显式 inline diffusion。 +- 强制：alias 由 `PipelineConfig.stage_cli_aliases` 唯一声明并在 structured/legacy factory 进入 stage merge 前归一；canonical `stage__` 与 alias 同时出现时，canonical 值优先并告警，alias 出现在其他 stage 必须拒绝。`prompt_transform_func`、`model_path_resolver` 与 `inline_diffusion` 由 `StagePipelineConfig` 投影到唯一 consumer；resolver 必须在构造 backend engine args 前消费并移除。inline 只允许单 replica，且须由 `inline_diffusion=true` 显式 opt in，或沿用已有 `custom_pipeline_args` 所声明的 in-process pipeline owner；其他 stage 默认保持 subprocess isolation。 +- 禁止：在 CLI、factory 或 stage startup 为模型名硬编码 alias/resolver；把 alias 广播到所有 stage；因 diffusion stage 只有一个 replica 就隐式改变所有多 stage pipeline 的进程隔离。 +- 验收：structured/legacy 两路覆盖 alias-only、canonical-only、相同/冲突值、错误 stage 与 unset control；用 root/partition/task path 证明 resolver 到达 stage model/tokenizer consumer，并用 opt-in 与相邻未 opt-in pipeline 证明 inline 只改变声明的 stage。^[PR #5885] diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index c338ee0f3..e4a050887 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -21,7 +21,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| | strict schema、unknown field、alias、flat→nested、structured/legacy/direct parity、typed projection | `strict-normalization`：`VOMNI-CFG-1a`–`1h` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,strip_parent_engine_args}` → `vllm_omni/config/omni_config.py::{_build_diffusion_config_projection,VllmOmniConfig.from_pipeline_config}` → `vllm_omni/engine/stage_init_utils.py::{build_engine_args_dict,build_engine_args_dict_from_omni_stage_config}` | -| deploy YAML、`base_config`、pipeline/stage overlay、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a` | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::{StageConfigFactory.create_from_model,StageConfigFactory._merge_cli_overrides}` | +| deploy YAML、`base_config`、pipeline/stage overlay、pipeline-owned alias/hook、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a`, [CONF-5i](rules-topology-profiles.md#conf-5i-pipeline-专属-alias-与-hook-必须在通用配置层声明归一并消费) | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,normalize_pipeline_cli_overrides,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::StageConfigFactory.create_from_model` → `vllm_omni/engine/stage_init_utils.py::_resolve_model_path` | | MiniCPM-o shipping profile、duplex/chat 共服、Talker sampling floor | model-specific [`MCPMO-4e`](../../models/minicpm-o-4-5/rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → `merge_pipeline_deploy`；duplex adapter runtime sampling override | | composable strategy、axis、routing、load balancing、`strategy-config` | `composable-strategy`：`CONF-4a` | `vllm_omni/config/composable_parallel/strategy_loader.py::{parse_strategy_specs,load_strategy_specs}` → `translator.py::translate_strategy_stack` → `apply.py::apply_strategy_specs` → `config_factory.py::{StageConfigFactory._apply_strategy_specs,StageConfigFactory._reconcile_strategy_with_cli}` | | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | @@ -35,7 +35,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `core` | 每次配置审查 | `VOMNI-CFG-1b`, `VOMNI-CFG-1c` | | `strict-normalization` | schema、alias、unknown field、structured/legacy/direct 路径 | `VOMNI-CFG-1a`, `VOMNI-CFG-1b`, `VOMNI-CFG-1c`, `VOMNI-CFG-1d`, `VOMNI-CFG-1e`, `VOMNI-CFG-1f`, `VOMNI-CFG-1g`, `VOMNI-CFG-1h` | | `deploy-memory` | 显存预算、KV pin、多 stage 共卡 | `CONF-1a`, `CONF-2a` | -| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5g` 见 [topology 与部署 profile](rules-topology-profiles.md)；MiniCPM-o profile 见 `MCPMO-4e` | +| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5i` 见 [topology 与部署 profile](rules-topology-profiles.md)；MiniCPM-o profile 见 `MCPMO-4e` | | `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a`, `CONF-4c`–`4d`，见 [并行拓扑合同](rules-parallel-topology.md) | | `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | | `stage-config-propagation` | pipeline sampling constraints、StageConfig serialization/runtime config | `VOMNI-CFG-1o` | diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9186153f4..c70abaa0f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ed837e35` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ b6a4baa6` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index a4864e35a..fca5f9f4e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308"] +sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308"] confidence: high --- @@ -102,12 +102,12 @@ confidence: high - 触发：修改 diffusion result pump、共享 `_rpc_futures`/`_output_futures`、`asyncio.wait_for()` 超时、请求 abort 或 shutdown future cleanup。\n- 强制：共享 Future 在 `done()` 检查后仍必须通过 `try_set_result`/`try_set_exception` 解析，并在 helper 内捕获 `concurrent.futures.InvalidStateError`，丢弃取消或已完成 Future 的晚到结果/异常，使 singleton pump 继续运行；仅 freshly-created 且尚未共享的本地 Future 可直接解析。\n- 禁止：把 `not fut.done()` 与 `set_result()`/`set_exception()` 当作原子操作；在共享 Future 上保留未保护的 resolve 调用；让取消竞态异常逃逸并杀死 result pump，随后仍以健康检查通过推断任务会完成。\n- 验收：用 `done()` 固定返回 false 但实际已取消的 racy Future 覆盖 `COMPUTE_DONE`、`OUTPUT_READY`、batch split 与 shutdown cleanup，断言 pump 不抛 `InvalidStateError`、取消 Future 保持取消、同批健康 Future 仍完成，并回归相邻并发测试。\n^[PR #5983] -## DIFF-1s — 非媒体 diffusion 输出必须显式闭合 output type 与 post-process +## DIFF-1s — diffusion output type 必须从模型声明闭合到 topology 与 formatter -- 触发：diffusion pipeline 产生 action 等非 image/video 输出，或修改 registry 中的 `final_output_type`、post-process hook 与 multiprocess 传递路径。 -- 强制：topology 的 `final_output_type` 必须与 pipeline 实际产物一致；非媒体输出使用稳定的 `DiffusionOutput.output` key，并注册可跨 orchestrator multiprocess 边界解析的 module-level post-process callable；没有变换时显式使用 identity hook。 -- 禁止：沿用 image 默认类型、用输出 ndarray 形状猜测语义、把 action 路由成 image/video，或把局部 closure 作为跨进程 post-process hook。 -- 验收：registry 能解析 pipeline 与 post-process callable，callable 可被目标进程边界使用，最终 engine output 保留稳定 key，并由端到端测试断言 action shape、dtype 和有限性。^[PR #4222] +- 触发：diffusion pipeline 产生 image/video/audio/action 等输出，或修改 model metadata、registry alias、topology `final_output_type`、formatter/post-process 与 multiprocess 传递路径。 +- 强制：model class 与 registry alias 解析到同一 canonical output metadata；默认单 stage config、multi-stage final stage、video endpoint capability 与 `OmniRequestOutput` formatter 必须传播同一类型。未知模型才回 image；非媒体输出使用稳定的 `DiffusionOutput.output` key，并注册可跨 orchestrator multiprocess 边界解析的 module-level post-process callable。 +- 禁止：只在 topology 标 video 但 formatter 仍发 image；要求每个 stage 都是 diffusion 才允许 video final；遗漏已注册 video alias 后静默回 image；用 ndarray 形状猜语义，或把局部 closure 作为跨进程 hook。 +- 验收：逐个 canonical class/alias 断言 default 与 multi-stage final type，video/image endpoint 正负 capability 及最终 output type；action 另验证 post-process callable 可跨目标进程边界使用，最终 output 保留 stable key、shape、dtype 和有限性。^[PR #4222] ^[PR #5885] ## DIFF-9a — 共享 PyAV 预构造帧 mux 必须闭合资源与音频时间零点 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 09f24ed2c..2cb708365 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -35,6 +35,7 @@ confidence: high | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、masked cross-attention、预算与 admission | `system-runtime`：`DIFF-4a`–`4j`, `DIFF-4v` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | | image/diffusion service time、forward time、missing-versus-zero、per-step metric | [`metrics-evidence`](rules-metrics.md)：`DIFF-5a`, `DIFF-5b` | output formatter/sampling metadata → stage stats aggregation → serving Prometheus emission | +| image/video/audio/action final type、registry alias、endpoint capability、formatter | [`output-runtime`](rules-output-lifecycle.md#diff-1s-diffusion-output-type-必须从模型声明闭合到-topology-与-formatter)：`DIFF-1s` | `model_metadata.py` → default/multi-stage config → video capability → `output_formatter.py` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a1f7dbad4..1f67d89d9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ed837e35` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b6a4baa6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e408314fa..0b2372df9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ed837e35` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b6a4baa6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3b9b8b715..746c59790 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ed837e35` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b6a4baa6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6dda046ae..f351402e9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ed837e35` 验证存在 +- 源码校验：以上路径均已在 `main @ b6a4baa6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md index 946fd1d6f..495c86590 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -4,13 +4,13 @@ created: 2026-09-04 updated: 2026-09-04 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #3805", "PR #5374", "PR #6598", vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/engine/orchestrator.py, vllm_omni/inputs/, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_invalid_audio_speech.py, tests/entrypoints/openai_api/test_serving_speech.py, "PR #5181", "PR #6182"] +sources: ["PR #3805", "PR #5374", "PR #5885", "PR #6598", vllm_omni/data_entry_keys.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/engine/orchestrator.py, vllm_omni/inputs/, tests/engine/test_async_omni_engine_input.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_invalid_audio_speech.py, tests/entrypoints/openai_api/test_serving_speech.py, "PR #5181", "PR #6182"] confidence: high --- # 请求输入合同 -`SERV-4a`–`SERV-4p`：公开请求字段的校验、限界与 owner。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口。 +`SERV-4a`–`SERV-4q`：公开请求字段的校验、限界与 owner。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口。 ## SERV-4a — 公开字段由 serving 显式拥有 @@ -132,3 +132,10 @@ engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)； - 禁止：只靠 protocol allowlist 防御嵌套或 extra 参数；把 overflow 作为 engine-wide fatal；捕获无关异常来掩盖程序错误；为统一范围而丢失原有业务下界。 - 验收：覆盖明确字段的越界 validation（含 speech token/seed 边界）和 mock dispatch 序列化 overflow；后者断言 400、request state 已清理、thread 仍存活。每个修改的公开整数字段须覆盖合法边界与非法一侧。^[PR #6598] - 边界与未合入建议：目标只捕获 `OverflowError`，不证明其他序列化失败的可恢复性。PR merge 后仍有两个未 resolve review threads：三个 protocol 文件重复定义 int64 constants，且负 TTS seed 的用户可见放宽未写入 PR body；前者是去重建议，后者可由 merged code/tests 与提交历史确认，但两者都不是本提交新增的 shared-constant 或 release-note 合同。 + +## SERV-4q — Stage-0 prompt transform 必须保留 downstream 原始视图并闭合临时 artifact 生命周期 + +- 触发：pipeline 用 `prompt_transform_func` 为 Stage 0 重写 prompt/media，同时 downstream stage 仍需原始请求，或 transform 创建 request-scoped 临时目录。 +- 强制：transform 只操作 Stage-0 copy；原始 prompt 保留给 downstream，并在 transform 前后保持同一 global request ID。只把已处理的 metadata 合并回原始视图，先移除 transform-owned stale keys。临时目录只通过内部 `REQUEST_ARTIFACT_DIRS_KEY` 交给 request state；preprocess、companion build、enqueue 前失败立即回收，admit 后由 orchestrator 在所有 terminal cleanup 路径回收，且内部 key 不得进入 stage payload。 +- 禁止：把 transformed prompt 当作 downstream 原始媒体、让旧 prepared descriptor 跨请求复用、在 ownership 已交给 orchestrator 后由 frontend 提前删除，或让异常路径泄漏转码目录。 +- 验收：覆盖 transform replacement/copy、request ID 与 metadata merge、成功 terminal/abort、preprocess/companion/enqueue 异常以及无 artifact control；断言 downstream 看见原始媒体+允许的 processed meta，目录恰好由当前 owner 回收，内部 key 未传输。^[PR #5885] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 58953c5b8..fb51be9d9 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -29,6 +29,7 @@ confidence: high | vLLM rebase、chat parser/renderer API moved or removed | `engine-lifecycle` | target-release `vllm.parser.*` / renderer API → `serving_chat.py` / `api_server.py`; remove a compatibility fallback only after the pinned base image is the target release ^[PR #6606] | | TTS model detection、`stage_keys`/`model_archs`、adapter priority/topology、legacy migration | `engine-lifecycle`：`SERV-5e` | `tts_adapters/__init__.py::{iter_tts_detectors,detect_tts_model_type,all_tts_stage_keys,tts_entry_stage_archs}` → `serving_speech.py::_find_tts_stage` | | request-level LoRA、AR-only/multistage、streaming input update、stage cardinality | `request-contract`：`SERV-4l` | `serving_chat.py` LoRA resolve → `AsyncOmni.generate` → `AsyncOmniEngine._build_add_request_message` → stage-0 input processor | +| Stage-0 prompt transform、downstream original prompt、临时 artifact ownership/cleanup | `request-contract`：[SERV-4q](rules-request-input.md#serv-4q-stage-0-prompt-transform-必须保留-downstream-原始视图并闭合临时-artifact-生命周期) | `AsyncOmniEngine._build_add_request_message` → `StageSubmissionMessage.request_artifact_dirs` → `OrchestratorRequestState` terminal cleanup | | caller sampling params、pipeline `sampling_constraints`、stage config runtime extraction | `request-contract`：`SERV-4o` | `OmniBase.resolve_sampling_params_list` → `_get_sampling_constraints_list` → `_apply_sampling_constraints` | | OpenAI integer bounds、msgpack overflow、request-local cleanup | `request-contract`：`SERV-4p` | protocol request models → orchestrator stage dispatch → `_fail_request_client_error` | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | @@ -47,7 +48,7 @@ confidence: high | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 见 engine lifecycle；`SERV-5s` 见 [app assembly](rules-app-assembly.md) | | `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、streaming raw terminal、event-driven orchestration、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5q`（见 [engine 生命周期规则](rules-engine-lifecycle.md)），`SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | -| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds | `SERV-4a`–`4h`, `SERV-4l`–`4p`，全部见 [请求输入合同](rules-request-input.md) | +| `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds、transform artifact ownership | `SERV-4a`–`4h`, `SERV-4l`–`4q`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 53fb3701c..6ba0234e3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -4,13 +4,13 @@ created: 2026-07-16 updated: 2026-09-04 type: guide tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/] +sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/, "PR #5885"] --- # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ed837e35`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b6a4baa6`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -35,10 +35,14 @@ adapter。已有专属知识 owner 可从 [models index](_index.md) 按名称进 | 注册点 | 位置 | 计数 | |---|---|---| -| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 88 个架构名 / 32 个模型族目录 | +| AR/omni 架构 | `model_executor/models/registry.py` `_OMNI_MODELS` | 89 个架构名 / 32 个模型族目录 | | Diffusion pipeline | `diffusion/registry.py` `_DIFFUSION_MODELS` | 65 条 pipeline / 40 个模型族目录 | -| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 58 个 key | -| Deploy YAML | `vllm_omni/deploy/*.yaml` | 87 份 | +| Pipeline（model_type） | `config/pipeline_registry.py` `OMNI_PIPELINES` | 59 个 key | +| Deploy YAML | `vllm_omni/deploy/*.yaml` | 89 份 | + +PR #5885 adds `MiniMaxH3TextEncoder`, the opt-in `minimax_h3_disaggregated` pipeline key, +and regular/Turbo deployment profiles. It preserves bare H3's fused single-stage fallback; +the split topology is selected only by its explicit deploy config. ^[PR #5885] PR #6089 新增 `nemotron_labs_voicechat_duplex.yaml`，因此只增加 deploy profile；它没有 增加 AR/diffusion registry architecture 或 `OMNI_PIPELINES` key。该 profile 是原生全双工 @@ -123,8 +127,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from ab8d45d3 \ - --to ed837e35 \ + --from ed837e35 \ + --to b6a4baa6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 876f93f87..a90d0d102 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] confidence: high --- @@ -28,6 +28,12 @@ confidence: high MMH3-2a/2b。^[PR #5914] - 生成合同固定为 24 FPS 视频与 32 kHz 音频；官方输出 duration、named ratio、768 short-edge 与 32-pixel canvas policy 在 request validation 阶段执行，而不是由 VAE 静默修正。 +- `minimax_h3_disaggregated` 是显式 deploy 选取的两 stage topology：Stage 0 的 + `MiniMaxH3TextEncoder` 用 native vLLM Qwen3-VL runner 产生 layer-50 residual hidden states + 和 token-role tags，Stage 1 不加载 text encoder、保留原始媒体并 inline 执行 H3 diffusion。 + 裸 checkpoint discovery 仍走 fused single-stage fallback；不可将 split profile 当成默认拓扑。 + `text_encoder_tp_size` 在该 profile 只是 Stage 0 `tensor_parallel_size` alias，stage-scoped + Stage 0 override 优先，Stage 1 的同名 override 被拒绝。^[PR #5885] ## 并行与加载约束 @@ -36,6 +42,10 @@ confidence: high 并且必须同时整除 Qwen3-VL 的 64 个 attention heads 与 8 个 KV heads。 - 单卡 accuracy 路径使用 CPU offload；多卡部署可使用 Ulysses、text-encoder TP、VAE tile/patch parallel 或 layerwise offload，但每个组合都必须按最终并行拓扑和设备数验证。 +- bundled split defaults bind Stage 0 to TP2/max-num-seqs 1 and Stage 1 to TP1 + USP4 + VAE + patch parallel 4 with `model_loaded.text_encoder: false`; regular uses 50 steps and the Turbo + LoRA profile uses five steps with its flow shifts. They are an H3-specific placement profile, + not a cross-node transport or general throughput claim.^[PR #5885] - H3 DiT 支持加载时 online FP8：默认覆盖 token-refiner/主 block 的 attention、MLP、 condition projection 和 AdaLN projection；video/audio patch、timestep MLP、最终输出头、 text encoder 与 VAE 保持 BF16/FP32。`ignored_layers` 按 DiT 内部的精确 runtime prefix diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md index 4ecb9cf48..6018a3e47 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5752", "PR #5829", "PR #5978", "PR #6555", "PR #6688", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/reference_video.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064"] +sources: ["PR #5752", "PR #5829", "PR #5885", "PR #5978", "PR #6555", "PR #6688", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/model_executor/models/minimax_h3/reference_video.py, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064"] confidence: high --- @@ -93,3 +93,28 @@ confidence: high - 强制：`_probe_video` 优先使用容器提供的 `nb_read_frames`/`nb_frames`，仅在元数据缺失时追加 `-count_frames` 扫描；Qwen 抽样必须将请求的 frame indices 交给一次 ffmpeg raw-video pipe，通过 `select` 只解码这些帧，完整回退也复用同一 ffmpeg RGB24 路径。解码前校验 frame count、宽高和 indices，必须在 reshape 前校验输出字节数等于 `frame_count × width × height × 3`。 - 禁止：为每个采样帧启动一次 ffmpeg、无条件对高码率 prepared stream 做完整扫描、在 decord 缺失时重新使用逐帧 PyAV 全量解码，或对不完整 raw-video 输出静默 reshape；不得把该 CPU 解码优化扩展成 MUSA/NPU 专用路径或通用 decoder 性能保证。 - 验收：contract tests 覆盖 selective/full ffmpeg command、`select` indices、decord 缺失回退、容器帧数直读与缺失时的 count-frames fallback、非法 metadata 和 incomplete output；在固定 prepared video 上将 selective 与 full RGB24 数组逐像素对照 PyAV reference，并分别核对 sampled frame 数、尺寸和 timestamps。^[PR #6064] + +## MMH3-2l — split text encoder 必须保持 H3 presentation、条件载荷与原始媒体的所有权 + +- 触发：`minimax_h3_disaggregated`、H3 Qwen3-VL prompt processor、Stage 0→1 bridge、prefix/sender + cache、reference-video preparation 或 encoder-free Stage 1。 +- 强制：Stage 0 只接收 H3 的 Qwen presentation；Stage 1 必须接收 original prompt/media 加上 + typed `text_encoder_output`，不得把 encoder-transformed prompt 当作 DiT input。conditioning 是 + hidden states `[tokens, 5120]` 与 token-role tags `[tokens]`；wire 上 tags 必须为 `[tokens, 1]` + 才能作为 per-token prefix-cache payload，Stage 1 恢复一维语义并在 request ID、completion count、 + tensor type/rank/shape 与二值 tags 边界 fail closed。H3 sender cache 必须 bypass：cache hit 只给 + generic processor misses，不能重建 media-dependent presentation IDs。 +- 强制：Ref2VA 的 Stage 0 preparation 要把 serialized prepared-video descriptor 和 artifact owner + 交给 original request；Stage 1 reuse the RGB stream for VAE/audio conditions, and terminal/error paths + must clean it. This avoids duplicate ffprobe/transcode while preserving raw media ownership at Stage 1. + `inline_diffusion` remains explicit pipeline opt-in; a single replica alone must not move another + multi-stage pipeline's diffusion engine into the orchestrator process. Encoder-free DLO must skip absent + components, and final-output validation follows declared final video metadata rather than requiring every + stage to be diffusion. +- 禁止：把 split route 自动发现为 H3 default、把 `text_encoder_tp_size` broadcast 到 Stage 1、用 + unvalidated positional output/tensor fields transfer conditioning，或把 H200/B300 single topology + observations扩展成 cross-node transport、throughput 或 all-profile compatibility evidence。 +- 验收：覆盖 fused fallback versus explicit split selection; exact H3 prompt IDs/tags including sender-cache + hits; chunked/prefix-cache token tags; malformed bridge payload/request mismatch; artifact reuse and cleanup; + resolver/local partition paths; Stage 0-only TP alias precedence/rejection; encoder-free DLO; explicit inline + isolation; and regular/Turbo default sampling profiles. ^[PR #5885] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 144f377ba..9f17040b2 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -27,6 +27,7 @@ confidence: high | Qwen3-VL encoder、NPU causal GQA、AddRMSNorm residual | `MMH3-1k` | encoder SDPA helper/residual call → NPU model patch → `torch_npu` op | | FL2VA keyframe、Ref2VA mixed reference/时域限界、shape/output matrix | [MMH3-2a](rules-media.md#mmh3-2a-taskreferenceshape-与多输出必须作为一个输入矩阵维护) | `pipeline_minimax_h3.py` → `reference_video.py` | | media limit、typed/multipart reference、HTTP 400、temp source | [MMH3-2b](rules-media.md#mmh3-2b-media-ingress-在解码前限界request-错误保持-http-400) | `api_server.py` → `serving_video.py` → `reference_video.py` | +| split text encoder、H3 presentation/tag、conditioning bridge、prepared artifact | [MMH3-2l](rules-media.md#mmh3-2l-split-text-encoder-必须保持-h3-presentation条件载荷与原始媒体的所有权) | Stage-0 prompt transform → `MiniMaxH3TextEncoder` → prefix-cache payload → Stage-1 H3 diffusion | | conditioned VAE、fixed seed、`fork_rng`、MUSA/device RNG | [MMH3-2c](rules-cache-task.md#mmh3-2c-conditioned-vae-的固定种子必须按实际设备隔离并恢复) | `pipeline_minimax_h3.py` condition encode caller → `vae.py::{encode_image,encode_video}` | | modular checkpoint、combined/partition task、两套 DiT、shared component | [MMH3-2d](rules-cache-task.md#mmh3-2d-modular-h3-的-task-selector-必须同步权重能力与所有-dit-lifecycle) | model index discovery → startup task selection → task-specific transformer/cache lifecycle | | request `quality`、lossless/high、dynamic Cache-DiT | [MMH3-2e](rules-cache-task.md#mmh3-2e-quality-映射只决定-request-的-cache-dit-目标) | request sampling → quality policy → request Cache-DiT runtime → denoise | From 69012b9fb621f61ff81cb4aa1e02a62653150d24 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:54:10 +0800 Subject: [PATCH 342/465] knowledge(vllm-omni): sync c201fd77 (PR #6349) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 17 ++++++++++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/hunyuan-video/_index.md | 6 +++--- .../models/hunyuan-video/architecture.md | 15 ++++++++++----- 12 files changed, 41 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 84218a1e6..df6e19904 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ed837e350e18315ef5e2930b1bf33458036a3afc + previous_audited_sha: b6a4baa659238e5e13a2f295430820a3ae36dde5 audited_ref: main - audited_sha: b6a4baa659238e5e13a2f295430820a3ae36dde5 + audited_sha: c201fd77c63a9041f13e757d9c796f08abf15bb3 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7efdb5f46..d30cc9808 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b6a4baa659238e5e13a2f295430820a3ae36dde5`. The canonical machine baseline +c201fd77c63a9041f13e757d9c796f08abf15bb3`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index c8ca7c9e4..f6723e1cc 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349"] --- # vLLM-Omni CI @@ -80,6 +80,21 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie artifacts; the JSON change itself does not establish a performance result or an active regression threshold. ^[PR #6613] +## HunyuanVideo-1.5 ready/merge and nightly coverage + +- The basic one-H100 CPU-offload T2V online-serving row is marked `core_model` and + `advanced_model`, and the ready and merge Buildkite jobs select it with the matching + run level. The CacheDiT + layerwise CPU-offload one-card row and CacheDiT + TP=2 + + VAE patch-parallel=2 (with VAE tiling) two-card row are `full_model` only, so their + function coverage remains in the nightly H100×2 lane. This is deliberately a + scope split, not evidence that the richer combinations run in ready/merge. ^[PR #6349] +- The dedicated nightly DFX job runs separate T2V and I2V JSON configs: random data, + ten prompts at concurrency one, seed 42, negative prompt, and 832×480 / 33-frame / + four-step / 24-fps requests. Each has a one-H100 baseline and a two-H100 + CacheDiT/TP2/VAE-patch-parallel=2/tiling case. It uploads result JSON and logs but + configures no threshold, so it is benchmark-report/artifact collection rather than + a performance regression gate. ^[PR #6349] + ## 目录内容 | 遇到什么 | 查看哪里 | diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fd7ab0dbc..9dbf764ab 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b6a4baa6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c201fd77` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c70abaa0f..18fc4a59e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b6a4baa6` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ c201fd77` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1f67d89d9..e498f0e7e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b6a4baa6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c201fd77` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0b2372df9..8b8f7193e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b6a4baa6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c201fd77` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 746c59790..aa2bf8aa2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b6a4baa6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c201fd77` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f351402e9..16f897856 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b6a4baa6` 验证存在 +- 源码校验：以上路径均已在 `main @ c201fd77` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6ba0234e3..87b6aa196 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b6a4baa6`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c201fd77`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -127,8 +127,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from ed837e35 \ - --to b6a4baa6 \ + --from b6a4baa6 \ + --to c201fd77 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md b/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md index 9675a14aa..f3247f6ce 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-video/_index.md @@ -1,15 +1,15 @@ --- title: "HunyuanVideo-1.5（T2V/I2V,meanflow 旗标蒸馏）" created: 2026-07-21 -updated: 2026-09-02 +updated: 2026-09-04 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #4645", vllm_omni/diffusion/models/hunyuan_video/, vllm_omni/deploy/hunyuan_video_15.yaml, vllm_omni/config/pipeline_registry.py] +sources: ["PR #4645", vllm_omni/diffusion/models/hunyuan_video/, vllm_omni/deploy/hunyuan_video_15.yaml, vllm_omni/config/pipeline_registry.py, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, "PR #6349"] --- # HunyuanVideo-1.5 -以下事实在 `main @ 5d44868e` 复核。与 +以下架构事实在 `main @ c201fd77` 复核。与 [hunyuan-image3](../hunyuan-image3/_index.md) 是**不同家族**（本家族纯 diffusion 视频,AR registry 无入口;image3 的结构见该页）。 diff --git a/knowledge/repos/vllm-omni/models/hunyuan-video/architecture.md b/knowledge/repos/vllm-omni/models/hunyuan-video/architecture.md index a497fb95f..31abceef1 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-video/architecture.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-video/architecture.md @@ -1,15 +1,15 @@ --- title: "HunyuanVideo-1.5 架构" created: 2026-07-21 -updated: 2026-08-10 +updated: 2026-09-04 type: architecture tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py, vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py, "PR #5969"] +sources: [vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py, vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #5969", "PR #6349"] --- # HunyuanVideo-1.5 架构 -事实在 `main @ b98b7c85` 复核;入口/变体速览见 [index](_index.md)。 +事实在 `main @ c201fd77` 复核;入口/变体速览见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -55,8 +55,13 @@ sources: [vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py ## 怎样验证功能、精度和性能 -pin 上只有**功能面**验证入口;无精度基线或性能 gate 证据,相关结论需另行 -实测。TP>1 的 54 块 DiT 未见 CI 覆盖（deploy 默认 TP1）。 +功能覆盖分层：基本单 H100 CPU-offload T2V online-serving case 在 ready/merge 的 +`core_model` 与 `advanced_model` lane 运行；CacheDiT + layerwise CPU-offload（1 卡）和 +CacheDiT + TP=2 + VAE patch parallel=2 + tiling（2 卡）均为 nightly `full_model` 覆盖。 +nightly 另跑 T2V/I2V DFX configs：每项为随机数据、10 prompts、concurrency 1、seed 42、 +negative prompt、832×480、33 frames、4 steps、24 fps，比较单 H100 和 2×H100 +CacheDiT/TP2/VAE patch-parallel=2/tiling 方案。它们只上传 JSON/log artifacts，未设置 +基线阈值；因此不是性能 gate，也不证明超出该配置与运行的性能或硬件结论。^[PR #6349] - 单测：`tests/diffusion/models/hunyuan_video/test_hunyuan_video_quant_config_propagation.py`; e2e `tests/e2e/online_serving/test_hunyuan_video_15_expansion.py`;示例 From fc9e9c21235c4465513eb74a86b823d269e644bf Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 4 Sep 2026 23:59:54 +0800 Subject: [PATCH 343/465] knowledge(vllm-omni): sync 15c6ab8f (PR #6704) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 19 ++++++++++++++- knowledge/repos/vllm-omni/ci/rules.md | 23 ++++++++++++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 11 files changed, 52 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index df6e19904..ba20a76ca 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b6a4baa659238e5e13a2f295430820a3ae36dde5 + previous_audited_sha: c201fd77c63a9041f13e757d9c796f08abf15bb3 audited_ref: main - audited_sha: c201fd77c63a9041f13e757d9c796f08abf15bb3 + audited_sha: 15c6ab8fa7cd12c55aa12ce9c2aedca25c2c016d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d30cc9808..31e7cccff 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c201fd77c63a9041f13e757d9c796f08abf15bb3`. The canonical machine baseline +15c6ab8fa7cd12c55aa12ce9c2aedca25c2c016d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index f6723e1cc..6298267fe 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-04 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704"] --- # vLLM-Omni CI @@ -95,6 +95,23 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie configures no threshold, so it is benchmark-report/artifact collection rather than a performance regression gate. ^[PR #6349] +## AMD CI stabilization boundary + +- AMD merge lanes may combine READY and MERGE definitions only through the bootstrap's + explicit `DEBUG_TEST_YAML=merge,ready` override; the merged YAML shares one build step + and preserves each suite as a named group. This is a maintainer debugging control, not + a change to the normal branch-based READY-versus-MERGE selection. +- Long MI300 diffusion shards carry explicit Buildkite timeouts, while a known long-running + Qwen3-TTS CustomVoice `async_chunk` Whisper-validation case is an 80-minute, + `NonBlocking` quarantined step. It must return to the blocking sharded lane only after + the AMD failure is resolved; a passing neighboring shard does not establish that result. +- ROCm memory assertions sample steady-state inference and report peak usage relative to + each run's initial device use, after cache cleanup. They therefore do not count retained + compiler/workspace allocations from a preceding topology as model memory, and the ROCm + layerwise-offload floor remains a conservative 512 MiB signal rather than a portability + or performance claim. Cleanup diagnostics use bounded `rocm-smi` calls because an + optional `amd-smi` CPER ioctl can remain uninterruptibly blocked. ^[PR #6704] + ## 目录内容 | 遇到什么 | 查看哪里 | diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 47d7b4007..2471cc8f0 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-04 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704"] confidence: high --- @@ -24,6 +24,7 @@ confidence: high | 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | | diffusion tiny builder、model settings、alignment exclusion、重模型 OOM | `OMNI-CI-2d` | `tests/model_tests/diffusion/{diff_model_builders,model_settings,test_alignment}.py` → common offline tests | | perf baseline、hardware label、DFX result artifact、assert-baseline | `OMNI-CI-3a` | `tests/dfx/conftest.py`、`tests/dfx/perf/scripts/run_benchmark.py`、`run_diffusion_benchmark.py`、`tests/dfx/perf/tests/**` | +| AMD/ROCm CI timeout、quarantine、diagnostic hang 或 memory assertion | `OMNI-CI-2f` | `.buildkite/amd/**` → target diffusion/TTS test → `tests/helpers/clean.py` | ## OMNI-CI-1a — 硬件 lane 必须真实收集并执行目标路径 @@ -186,6 +187,26 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload - 验收：image tag 与目标 release 一致；startup 通过共享内存 preflight；目标 lane 证明实际 grader device；每个残余失败独立记录，尤其数值/质量回归不能由 API 兼容测试替代。^[PR #5976] +## OMNI-CI-2f — AMD CI stabilization must preserve bounded execution and the measurement signal + +- 触发：AMD/ROCm Buildkite bootstrap、long-running shard timeout/quarantine、GPU-memory assertion，或 + cleanup diagnostics 变更。 +- 强制：debug-only READY+MERGE suite composition must retain the normal branch selection and render one + shared build dependency with named groups; long-running MI300 steps need explicit Buildkite bounds. A known + failing case stays separately `NonBlocking` and time-bounded until its exact failure is resolved, rather than + remaining inside a blocking shard. +- 强制：memory comparisons start sampling after model/offload construction, subtract each run's initial device + usage, and clean retained cache state between runs. ROCm offload thresholds must remain conservative signals for + the asserted mechanism, not portability, capacity, or performance claims. Optional cleanup diagnostics must use + a bounded command that cannot turn a test failure into an unbounded hang. +- 禁止：把 READY/MERGE debug override 当成默认 lane routing；用邻近 shard 的绿色结果解除已知 case 的 + quarantine；把 absolute device use（含前一 topology 的 compiler/workspace cache）当模型峰值；或让 + diagnostic-only `amd-smi` hang bypass subprocess timeout and block pytest teardown. +- 验收：render `merge`、`ready`、combined and malformed/duplicate debug selections; prove MI300 long shards + have explicit timeouts and the unresolved TTS case is non-blocking. On ROCm, check the selected test's per-run + baseline/steady-state sampling and cleanup command separately from the model assertion; a local or different + machine threshold result is not a CI-wide performance verdict. ^[PR #6704] + ## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 - 触发：修改 perf JSON 的 `baseline`、硬件 label/marker、benchmark result schema，或恢复性能阈值断言。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9dbf764ab..1c2ef141c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c201fd77` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 15c6ab8f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 18fc4a59e..97371a8b3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c201fd77` 验证存在；本轮新增/扩展的 +- 源码校验：以上子模块均已在 `main @ 15c6ab8f` 验证存在；本轮新增/扩展的 async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e498f0e7e..ab38d9a45 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c201fd77` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 15c6ab8f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8b8f7193e..a99857a52 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c201fd77` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 15c6ab8f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index aa2bf8aa2..5e6e60432 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c201fd77` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 15c6ab8f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 16f897856..c64a1ff45 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c201fd77` 验证存在 +- 源码校验：以上路径均已在 `main @ 15c6ab8f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 87b6aa196..47118004c 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c201fd77`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 15c6ab8f`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -127,8 +127,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from b6a4baa6 \ - --to c201fd77 \ + --from c201fd77 \ + --to 15c6ab8f \ --repo \ --mode report-only ``` From fe5c653f5fde191046f2ab64567d9c65bb07827a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 00:11:49 +0800 Subject: [PATCH 344/465] knowledge(vllm-omni): sync 04b97a59 (PR #6607) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 6 ++-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 8 ++--- .../vllm-omni/models/minimax-h3/_index.md | 10 +++++- .../models/minimax-h3/rules-vae-ops.md | 36 +++++++++++++++++++ .../vllm-omni/models/minimax-h3/rules.md | 3 +- 12 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/rules-vae-ops.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ba20a76ca..bbe4d1c80 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c201fd77c63a9041f13e757d9c796f08abf15bb3 + previous_audited_sha: 15c6ab8fa7cd12c55aa12ce9c2aedca25c2c016d audited_ref: main - audited_sha: 15c6ab8fa7cd12c55aa12ce9c2aedca25c2c016d + audited_sha: 04b97a5922493f78293623d6169e62ccecde9116 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 31e7cccff..7217af52d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -15c6ab8fa7cd12c55aa12ce9c2aedca25c2c016d`. The canonical machine baseline +04b97a5922493f78293623d6169e62ccecde9116`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1c2ef141c..a0fa9b916 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 15c6ab8f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 04b97a59` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 97371a8b3..f7421d21d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -1,7 +1,7 @@ --- title: "Diffusion" created: 2026-07-10 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, components, diffusion] sources: [] @@ -10,8 +10,8 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 15c6ab8f` 验证存在；本轮新增/扩展的 - async output、distributed layerwise offload、MiniMax H3 和 SANA-Video 仍按各自模型/机制规则审查 +- 源码校验：以上子模块均已在 `main @ 04b97a59` 验证存在；MiniMax-H3 的 VAE eager + ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ab38d9a45..315f8568a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 15c6ab8f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 04b97a59` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a99857a52..061aebd86 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 15c6ab8f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 04b97a59` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 5e6e60432..f6cf1de72 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 15c6ab8f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 04b97a59` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c64a1ff45..a50825161 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 15c6ab8f` 验证存在 +- 源码校验：以上路径均已在 `main @ 04b97a59` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 47118004c..6bc7ea331 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -1,7 +1,7 @@ --- title: "模型代码入口与 registry 快照" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: guide tags: [vllm-omni, models] sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/, "PR #5885"] @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 15c6ab8f`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 04b97a59`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -127,8 +127,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from c201fd77 \ - --to 15c6ab8f \ + --from 15c6ab8f \ + --to 04b97a59 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index a90d0d102..7375dffd1 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -1,7 +1,7 @@ --- title: "MiniMax H3" created: 2026-08-05 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] @@ -70,6 +70,11 @@ confidence: high 4D batch layout，再恢复原 shape。MUSA dynamic compile 独立保留 aten RMSNorm graph 并内联 full-dim NeoX RoPE，静态 rot width 从 config 派生；平台 fallback、共享 blast radius 与有界 region-only 性能证据见 MMH3-1c/DIFF-1e。 +- H3 VAE decoder 的 exact eager ops 仍由 model owner 管理：只有 SM90/SM100/SM103 allowlist、 + remote-code structure 和每个 tensor guard 都满足时才安装。仅 decoder Transformer-block + Linear 固化为 decode autocast 已使用的 FP16；keyframe encode 与 `proj_out` 等非 block + parameter 继续 FP32。compile、spatial-parallel 及任何 platform/shape/dtype/layout/gradient + guard 均回原 operation；完整合同见 MMH3-4c。^[PR #6607] - Ascend NPU 可选择 RainFusion 稀疏 video tail，并从 BF16 checkpoint 做 online INT8；两者只在 exact T2VA/Ulysses/no-AllGather DLO 配置有完成证据，几何 fallback、TP width 与组合边界见 MMH3-1a/1d。 @@ -134,3 +139,6 @@ H3 input matrix/media ingress，以及 text-encoder completeness、online FP8 [media rules](rules-media.md) 与 [MiniMax H3 rules](rules.md#direct-代码快速入口)；checkpoint transform、quantized loader 与 text-encoder fused-source 完整性正文见 [loading rules](rules-loading.md)；DLO、consumer/H100/ROCm 部署和硬件证据正文见 [deployment rules](rules-deployment.md)；conditioned VAE 确定性、modular task 选择与 request 级 Cache-DiT/TeaCache/sigma schedule/Turbo LoRA 生命周期见 [缓存与任务生命周期规则](rules-cache-task.md)。 + +H3 VAE decoder 的 model-local eager dispatch、remote-code eligibility、exactness guards、selective +FP16 materialization 与 compile/spatial-parallel fallback 见 [VAE eager-ops rules](rules-vae-ops.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-vae-ops.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-vae-ops.md new file mode 100644 index 000000000..a2b17db66 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-vae-ops.md @@ -0,0 +1,36 @@ +--- +title: "MiniMax H3 VAE eager-ops rules" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #6607", vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/models/minimax_h3/ops/README.md, vllm_omni/diffusion/models/minimax_h3/ops/vae/__init__.py, vllm_omni/diffusion/models/minimax_h3/ops/vae/dispatch.py, vllm_omni/diffusion/models/minimax_h3/ops/vae/qk_norm_rope.py, vllm_omni/diffusion/models/minimax_h3/ops/vae/scaled_residual.py, tests/diffusion/models/minimax_h3/test_minimax_h3_vae_ops.py] +confidence: high +--- + +# MiniMax H3 VAE eager-ops rules + +## MMH3-4c — H3 VAE eager ops 必须以完整远程模型合同和 reference fallback 安装 + +- 触发：修改 `MiniMaxH3VideoVAE`、其 model-local `ops/vae/` dispatch、decoder block patch、 + FP16 materialization 或 VAE exactness tests。 +- 强制：只在 CUDA+Triton、非 HIP、Omni CUDA platform available 且 compute capability 精确为 + SM90、SM100 或 SM103 时，从 flat operator-set table 选一个完整实现；其他 capability 保持 + reference。安装前必须验证 official remote decoder 的 block/linear shapes、RMSNorm/FP32 QK-norm + semantics、gated SiLU FF、scale tensor、non-compiled FF state 和 spatial-parallel flag；不匹配 + 不改变模块。安装只能一次，并且只将 transformer-block 的 `to_qkv`、`to_out`、`w1`、`w2` + FP32 weights materialize 为 FP16；keyframe encode 与 block 外 decoder parameters（包括 + `proj_out`）保持 FP32。 +- 强制：融合 Q/K norm+RoPE 和 scaled residual 都是 exact contract：前者只收 CUDA contiguous + FP16 Q/K `[B,S,H,64]` 与 contiguous FP16 `[B,S,1,48]` cos/sin，保持 FP32 RMS reduction 和 + 显式 FP16 multiply/add rounding；后者只收 contiguous FP32 residual、FP16 branch、FP32 scale， + hidden width 2048，并禁止 FMA contraction。任一 shape/dtype/device/layout/gradient guard 不满足 + 必须返回 `None`，由原 operation 执行。 +- 禁止：在 `torch.compile` 或 spatial-parallel attention 使用 patch；把 model-local kernel 提升为 + shared operator proof，或由 SM90/SM103 exactness/latency evidence外推 SM100、其他 GPU、task、 + tile/parallel topology 或 E2E benchmark。新 backend 应新增 complete operator-set entry 并重做其 + platform evidence，不能放宽现有 allowlist。 +- 验收：CPU/mock contract tests 覆盖 all-target dispatch、unsupported untouched、remote-structure + rejection、idempotence、selective FP16 and FP16 SwiGLU path、compile/SP/original-forward fallback； + target GPU 另以 direct Q/K、residual 与 complete decoded tensor 的 `torch.equal` 对 reference + 验 exactness。性能报告必须绑定 exact device/software/workload，不能代替持续 CI gate。^[PR #6607] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 9f17040b2..4c0e2be75 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -1,7 +1,7 @@ --- title: "MiniMax H3 规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", "PR #6040", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526"] @@ -25,6 +25,7 @@ confidence: high | text encoder、missing q/k/v 或 gate/up、eager load bookkeeping | [MMH3-1f](rules-loading.md#mmh3-1f-text-encoder-eager-load-必须证明每个-source-shard-完整) | `encoder.py::_load_weights` → `pipeline_minimax_h3.py::load_weights` strict report | | NPU packed varlen、quadratic mask、LaserAttention、prefix K/V | `MMH3-1g` | H3 packed producer → backend capability/metadata → NPU FlashAttention fallback | | Qwen3-VL encoder、NPU causal GQA、AddRMSNorm residual | `MMH3-1k` | encoder SDPA helper/residual call → NPU model patch → `torch_npu` op | +| VAE decoder、Triton exact ops、FP16 materialization、SM90/100/103、compile/SP fallback | [MMH3-4c](rules-vae-ops.md#mmh3-4c-h3-vae-eager-ops-必须以完整远程模型合同和-reference-fallback-安装) | `vae.py::MiniMaxH3VideoVAE.__init__` → `ops/vae::{dispatch,install}` → target tests | | FL2VA keyframe、Ref2VA mixed reference/时域限界、shape/output matrix | [MMH3-2a](rules-media.md#mmh3-2a-taskreferenceshape-与多输出必须作为一个输入矩阵维护) | `pipeline_minimax_h3.py` → `reference_video.py` | | media limit、typed/multipart reference、HTTP 400、temp source | [MMH3-2b](rules-media.md#mmh3-2b-media-ingress-在解码前限界request-错误保持-http-400) | `api_server.py` → `serving_video.py` → `reference_video.py` | | split text encoder、H3 presentation/tag、conditioning bridge、prepared artifact | [MMH3-2l](rules-media.md#mmh3-2l-split-text-encoder-必须保持-h3-presentation条件载荷与原始媒体的所有权) | Stage-0 prompt transform → `MiniMaxH3TextEncoder` → prefix-cache payload → Stage-1 H3 diffusion | From 982e9ef0f1317f5a402c5646b283f8c1cc742f1a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 00:25:07 +0800 Subject: [PATCH 345/465] knowledge(vllm-omni): sync 009b992c (PR #6170) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 11 +++++++-- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../serving/rules-fault-isolation.md | 9 +++++-- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minicpm-o-4-5/rules.md | 24 +++++++++++++------ 12 files changed, 45 insertions(+), 23 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bbe4d1c80..caecd108f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 15c6ab8fa7cd12c55aa12ce9c2aedca25c2c016d + previous_audited_sha: 04b97a5922493f78293623d6169e62ccecde9116 audited_ref: main - audited_sha: 04b97a5922493f78293623d6169e62ccecde9116 + audited_sha: 009b992c5b5d36acc55759dba7ed5acfee90d324 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7217af52d..e0eebef4f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -04b97a5922493f78293623d6169e62ccecde9116`. The canonical machine baseline +009b992c5b5d36acc55759dba7ed5acfee90d324`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 6298267fe..14c10e871 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni CI" created: 2026-07-10 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704"] --- # vLLM-Omni CI @@ -13,6 +13,13 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie - 处理 vLLM-Omni 的 L2/L4、模型测试配置或 CI 特有问题。 +## MiniCPM native-duplex server-VAD coverage + +- Ready/merge 的 MiniCPM duplex source dependencies 纳入 server-VAD runner；其 L4 full-model + acceptance 验证 one hard-cancel terminal、无旧 epoch delta、speech edges、interrupt PCM/pre-roll + 与 follow-up response。它是 protocol/lifecycle coverage，不能单凭 runner 或历史 H200 结果声称 + 常规 listen-only 性能提升。 ^[PR #6170] + ## 不放什么 - 跨仓库通用的测试方法。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a0fa9b916..fcb707769 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 04b97a59` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 009b992c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f7421d21d..2b78ab902 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 04b97a59` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 009b992c` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 315f8568a..247a4bee5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 04b97a59` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 009b992c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 061aebd86..0e4893371 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 04b97a59` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 009b992c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f6cf1de72..bbce7a6ac 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 04b97a59` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 009b992c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a50825161..d493abf94 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 04b97a59` 验证存在 +- 源码校验：以上路径均已在 `main @ 009b992c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-fault-isolation.md b/knowledge/repos/vllm-omni/components/serving/rules-fault-isolation.md index 3cb08da16..425813d54 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-fault-isolation.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-fault-isolation.md @@ -1,10 +1,10 @@ --- title: "Serving replica fault-isolation 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4583", vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py] +sources: ["PR #4583", "PR #6170", vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py] confidence: high --- @@ -54,3 +54,8 @@ confidence: high 计数、comprehension/diffusion metadata 查找和 PD bootstrap metadata 仍读 init-time `stage_clients`，动态成员语义需另行收敛。本 pin 没有新增与 readiness 分开的 liveness endpoint 或部署文档。^[PR #4583] + +- duplex inter-stage `process_engine_inputs` 失败同样只能成为目标 request 的结构化 stage error： + 发送 error 后以 abort/close-duplex-session cleanup 回收该 request；普通非-duplex 路径继续抛出， + 不能扩大 catch 范围而吞没共享 orchestrator 的编程错误。验收要证明坏 duplex handoff 不终止 + orchestrator，且同一 engine 的后续 request 仍可完成。^[PR #6170] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6bc7ea331..d520eeb77 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 04b97a59`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 009b992c`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -127,8 +127,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 15c6ab8f \ - --to 04b97a59 \ + --from 04b97a59 \ + --to 009b992c \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 0c932e8f3..65d1073f1 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -1,10 +1,10 @@ --- title: "MiniCPM-o 4.5 规则" created: 2026-07-20 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619"] confidence: high --- @@ -207,14 +207,24 @@ confidence: high - 触发：MiniCPM-o native duplex 的连续音频、LISTEN/SPEAK handoff 或 server VAD interruption。 - 强制：模型说话时仍把音频单元追加到同一可恢复 Stage0，保留 KV/runtime state 和上一 - terminator；LISTEN 是成功且不需要 Talker hidden state，SPEAK 才转交 Talker。server VAD - hard cancellation 必须显式 opt-in，阈值不得落到使 silence branch 不可达的边界。 + terminator；仅确认的 empty/control-only handoff 可在没有 speech conditioning 时跳过，其他 + 缺 latent/hidden-state 的 speech handoff 仍报错。LISTEN 是成功且不需要 Talker hidden state， + SPEAK 才转交 Talker。默认模型拥有的模式固定为 `listen_only`，不会创建/调用 Silero；只有 + `turn_detection.type=server_vad` 且 `interrupt_response=true` 才选择 `barge_in_on_speech`， + 并拒绝 `interrupt_response=false` 的第三模式。server VAD hard cancellation 必须显式 opt-in， + 阈值不得落到使 silence branch 不可达的边界。 - 禁止：用 server VAD cancellation 冒充 native interruption；每轮重建 Stage0；让 LISTEN 因 - 无 Talker payload 失败。 + 无 Talker payload 失败；将 Qwen3-Omni 的 turn-based server-VAD endpointing/auto-commit + 合同混同为本 MiniCPM native-barge-in 行为。 - 验收：native case 完成响应≥2、cancel/truncate=0、listen≥1；opt-in hard interrupt 恰好一次 - terminal、fence 后无 stale delta、保留打断语句且后续响应成功。response-required fixture 若要求 + terminal、epoch fence 后无 stale delta，且新 epoch 保留 interrupt PCM/confirming pre-roll、 + 后续响应消费该语句并成功。Realtime `session.update` 先暂存 VAD candidate，只有 matching + runtime ACK 才 commit，任何 rejected/failed append/config path 都必须 correlated error 后 + discard，不能让 unrelated outbound error 提交它或让 reader 卡住。response-required fixture 若要求 每轮 SPEAK，应重放已验证的完整 active-speech window；任意较短 mid-utterance slice 合法返回 - LISTEN，不能把它当模型失败或修改生产决策阈值。 ^[PR #6056] ^[PR #6154] ^[PR #6170] + LISTEN，不能把它当模型失败或修改生产决策阈值。`threshold=0.15` 的 validator boundary assertion + 在 merge 时缺失，是后续测试补强，不是本提交已实现的覆盖；#6821 的 playback ACK/history 修复 + 也在本精确提交之外。 ^[PR #6056] ^[PR #6154] ^[PR #6170] ## MCPMO-4d — async chunk 快照替换必须清理旧音频 From 1f3e287032503da667567a634ad04eadd3bfb437 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 00:33:44 +0800 Subject: [PATCH 346/465] knowledge(vllm-omni): sync 13173d6b (PR #6307) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 ++- .../diffusion/rules-worker-observability.md | 18 ++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index caecd108f..a3cb2475d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 04b97a5922493f78293623d6169e62ccecde9116 + previous_audited_sha: 009b992c5b5d36acc55759dba7ed5acfee90d324 audited_ref: main - audited_sha: 009b992c5b5d36acc55759dba7ed5acfee90d324 + audited_sha: 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e0eebef4f..462495dca 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -009b992c5b5d36acc55759dba7ed5acfee90d324`. The canonical machine baseline +13173d6bbbd0f0ddd4f6caa2debabcf2375d911d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index fcb707769..99aa66dfc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 009b992c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2b78ab902..ace21f342 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 009b992c` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 13173d6b` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -31,6 +31,7 @@ sources: [] | 理解共享职责和数据流 | [architecture](architecture.md) | | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | image/diffusion timing、step normalization、missing-versus-zero 与 benchmark 解释边界 | [metrics evidence rules](rules-metrics.md) | +| worker process title、topology rank 与日志前缀 | [worker observability rules](rules-worker-observability.md) | | vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md b/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md new file mode 100644 index 000000000..90b10069d --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md @@ -0,0 +1,18 @@ +--- +title: "Diffusion worker observability rules" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #6307", vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_diffusion_worker_process_title.py] +confidence: high +--- + +# Diffusion worker observability rules + +## DIFF-4o — worker 进程标题和日志前缀必须来自同一已初始化拓扑 + +- 触发：修改 `DiffusionWorker`/`WorkerProc` 启动、model-parallel 初始化，或 worker 的进程标题和日志装饰。 +- 强制：early startup 先用通用 `DiffusionWorker` 标识；`initialize_model_parallel()` 完成后、模型加载前再次从 active group 生成同一个标识并同时交给 upstream `set_process_title(..., prefix="vLLM-Omni")` 与 `decorate_logs()`。按 DP、PP、SP、CFG、TP、FS（HSDP）顺序附加非 singleton group 的 `_`；仅在 expert parallel enabled 时查询 EP 并附加它。所有 singleton 维度省略。 +- 禁止：用 global rank 代替局部 topology rank；在 parallel state 可用前读取 group；仅更新 OS title 或仅更新日志前缀；因 optional `setproctitle` 缺失让 worker 启动失败；把该可观测性改动描述为 scheduling、通信、模型输出或性能路径变化。 +- 验收：覆盖初始化前 generic 名称及不读取 group、singleton、DP/PP/SP/CFG/TP、standalone FS/HSDP、conditional EP、缺少 `setproctitle` 仍装饰日志，以及 Linux `ps` 可见 title；冷启动 `ps` polling 要有足够 wall-clock budget，避免 import 延迟造成 flaky test。此状态的 `get_fs_group` 依赖在随后 parallel-state 改版中会不兼容，属于本提交之后的边界，不能回写为本提交未实现。^[PR #6307] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 2cb708365..88a9b9311 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -1,7 +1,7 @@ --- title: "Diffusion 共享规则" created: 2026-07-20 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 247a4bee5..9a0e1977c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 009b992c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0e4893371..0a8e4768f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 009b992c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bbce7a6ac..e2428d59b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 009b992c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d493abf94..aaf71bf1f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 009b992c` 验证存在 +- 源码校验：以上路径均已在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d520eeb77..bd0894b56 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 009b992c`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 13173d6b`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -127,8 +127,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 04b97a59 \ - --to 009b992c \ + --from 009b992c \ + --to 13173d6b \ --repo \ --mode report-only ``` From 4eaf32b374428a14271b3253b4b2888de41f8c2f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 00:42:40 +0800 Subject: [PATCH 347/465] knowledge(vllm-omni): sync da94c3eb (PR #6404) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/benchmark/overview.md | 6 ++++-- knowledge/repos/vllm-omni/ci/_index.md | 4 ++++ .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minicpm-o-4-5/_index.md | 6 ++++-- .../vllm-omni/models/minicpm-o-4-5/rules-duplex.md | 12 ++++++++++-- 13 files changed, 34 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a3cb2475d..b94dd0c7b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 009b992c5b5d36acc55759dba7ed5acfee90d324 + previous_audited_sha: 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d audited_ref: main - audited_sha: 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d + audited_sha: da94c3ebe15c78f5c580936f6b18a6d97d03539b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 462495dca..1f309b71b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -13173d6bbbd0f0ddd4f6caa2debabcf2375d911d`. The canonical machine baseline +da94c3ebe15c78f5c580936f6b18a6d97d03539b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/benchmark/overview.md b/knowledge/repos/vllm-omni/benchmark/overview.md index a6ecbf720..fc031e162 100644 --- a/knowledge/repos/vllm-omni/benchmark/overview.md +++ b/knowledge/repos/vllm-omni/benchmark/overview.md @@ -1,10 +1,10 @@ --- title: "Benchmark Notes" created: 2026-07-10 -updated: 2026-09-04 +updated: 2026-09-05 type: guide tags: [vllm-omni, benchmark] -sources: ["PR #6634", vllm_omni/benchmarks/duplex/, vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py, vllm_omni/experimental/fullduplex/client.py] +sources: ["PR #6404", "PR #6634", vllm_omni/benchmarks/data_modules/daily_omni_dataset.py, vllm_omni/benchmarks/duplex/, vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py, vllm_omni/experimental/fullduplex/client.py] --- # Benchmark Notes @@ -24,3 +24,5 @@ scoring-protocol and evidence contracts are documented in the [Omni-DuplexEval guide](guides/omni-duplex-eval.md). ^[PR #6634] 具体 PR 的临时复测、讲解稿和 PR body 数据不再作为长期 benchmark 文档保留。这里仅路由可跨 PR 复用的 workload、口径、artifact 和诊断经验。 + +Daily-Omni 的 pack-mode 兼容测试随 PR #6404 调整，但没有新增 registry、pipeline 或 benchmark 结果；video frame-to-unit pairing 是 Realtime client/Stage 0 correctness contract，而不是新的 benchmark metric 或性能结论。^[PR #6404] diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 14c10e871..4afdf939a 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -20,6 +20,10 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie 与 follow-up response。它是 protocol/lifecycle coverage，不能单凭 runner 或历史 H200 结果声称 常规 listen-only 性能提升。 ^[PR #6170] +## MiniCPM Realtime video-frame unit coverage + +- PR #6404 adds CPU/unit coverage for closing-append frame placement, held last/still frames, two-frame stacking, Stage 0 marker/embedding ordering, rejected early video appends, dtype conversion, and external WAV video demos. It is protocol and input-assembly coverage; its unchecked full-model video E2E and manual demo do not establish an end-to-end checkpoint pass or video quality result. ^[PR #6404] + ## 不放什么 - 跨仓库通用的测试方法。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 99aa66dfc..a78d2968b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ace21f342..8c5373599 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 13173d6b` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ da94c3eb` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9a0e1977c..62ba3e4ca 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0a8e4768f..679979532 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e2428d59b..de02238ed 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index aaf71bf1f..fb1e4d7c0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d` 验证存在 +- 源码校验：以上路径均已在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index bd0894b56..e5cbdb94e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 13173d6b`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ da94c3eb`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -127,8 +127,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 009b992c \ - --to 13173d6b \ + --from 13173d6b \ + --to da94c3eb \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index f33e6d8c5..090b3b384 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -1,10 +1,10 @@ --- title: "MiniCPM-o 4.5" created: 2026-07-20 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6678", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082", "PR #6587"] +sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6404", "PR #6678", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082", "PR #6587"] confidence: high --- @@ -55,6 +55,8 @@ Talker chunk、shipping profile 与 chat 隔离见 [native duplex 规则](rules- state、batch cap 与 NPU residual limits 见 [Code2Wav 并发批处理规则](rules-code2wav-batching.md)。新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 +Realtime video 不以「每 200 ms append」定义 frame cadence：首个约 1030 ms、以后每秒关闭的 Stage 0 unit 才接收对应 camera frame；双图 stacking 与外部 WAV 覆盖 soundtrack 的边界见本页已链接的 native duplex 规则 MCPMO-4f。^[PR #6404] + ## 什么时候查这里 - 审查 MiniCPM-o 4.5 registry、remote-code gate、TTS dependency、Code2Wav TensorRT/HiFT/CFM graph、 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md index c79145034..c30c66c9c 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -1,10 +1,10 @@ --- title: "MiniCPM-o 4.5 native duplex 规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6318", "PR #6346", "PR #6458", "PR #6619", "PR #6630", "PR #6678", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] +sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] confidence: high --- @@ -57,3 +57,11 @@ confidence: high 与未声明 `duplex_session` 时的 server 语义保持一致。CPU 守卫至少覆盖 shipping base 的 显式容量、未声明字段的 default，及一个从 base 继承容量的 overlay；live probe 再验证在 该 limit 后收到 capacity error。^[PR #6678] + +## MCPMO-4f — omni video frame 必须随其关闭的 audio unit 原子进入 Stage 0 + +- 触发：修改 MiniCPM-o Realtime duplex 的 camera/video append、Stage 0 streaming prefill、vision embedding，或视频 demo/client 的 audio-frame interleave。 +- 强制：client 以累计 appended PCM 的 unit boundary 绑定 frame `k`：首个 unit 在约 `1030 ms`（`first_chunk_ms=1035` 对齐 160-sample hop 后），后续每 `1000 ms`；frame 只能附在**关闭 unit `k`**的 append，不能在早期 append 暂存或跨 append carry。视频短于音频时保持最后一帧，单张 still 同样重复。`stack_frames=2` 是 previous+current 的两张图；整个 append 的 `frame_list` 都属于它关闭的一个 unit，顺序为 ``、每张各自 `` + 64 vision embeddings + ``、再 audio。vision hidden states 在拼入 token embedding 前转为 token-embedding dtype。 +- 拒绝：携 frame 但没有关闭 model unit 的 append 必须显式失败，且不推进 `audio_chunk_idx`；不能只丢帧后继续音频。Realtime wire 每 append 最多两张图，stacked composite 不改变一秒 audio cadence。视频 demo 同时传 `--input-wav` 和 `--input-video` 时，外部 WAV 覆盖视频 soundtrack 且决定 frame timeline/duration。 +- 验收：覆盖 frame 0 不早于约 1030 ms、每个 frame `k` 绑定其 closing append、无跨 append carry、short/still hold-last、`[f0]`/`[f0,f1]`/`[f1,f2]` stacked 序列、双图 marker/128 embedding 布局、no-unit rejection 不推进 audio index、dtype 对齐，以及 silent video + external WAV。Daily-Omni pack-mode 测试只覆盖数据包装边界，不能替代 Realtime session 或 Stage 0 验证。^[PR #6404] +- 已知边界：Stage 0 vision encode 失败虽返回 failed prefill 且不推进 `audio_chunk_idx`，但 `preprocess()` 仍可能 fallback 到原 input ids/embeddings；保留的 scheduler span/KV 或 `num_computed_tokens` 是否推进尚未 abort/rollback 证明。另一个 fast/malformed client 的 pending-frame queue 仍未封顶。两者都是未合入 follow-up，不得写成此 frame-binding 修复已解决。此前 NPUGraph / soft-interrupt 范围已从本 PR 移除。^[PR #6404] From dca3850658d186c5f825db4bb34be204f7c2e48d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 00:50:06 +0800 Subject: [PATCH 348/465] knowledge(vllm-omni): sync 4ad455e6 (PR #6001) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/rules.md | 3 ++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/distributed/async-chunk.md | 7 +++++-- .../vllm-omni/components/distributed/rules.md | 10 ++++++++-- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 3 ++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- .../vllm-omni/models/qwen3-tts/_index.md | 6 +++--- .../repos/vllm-omni/models/qwen3-tts/rules.md | 19 ++++++++++++++++--- 15 files changed, 47 insertions(+), 23 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b94dd0c7b..b700c088c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 13173d6bbbd0f0ddd4f6caa2debabcf2375d911d + previous_audited_sha: da94c3ebe15c78f5c580936f6b18a6d97d03539b audited_ref: main - audited_sha: da94c3ebe15c78f5c580936f6b18a6d97d03539b + audited_sha: 4ad455e61e2857450594189895fe860596f402dc # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1f309b71b..8659b9e37 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -da94c3ebe15c78f5c580936f6b18a6d97d03539b`. The canonical machine baseline +4ad455e61e2857450594189895fe860596f402dc`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a78d2968b..7762afd0b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index e4a050887..04a6ebada 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni 配置开发门禁" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, config] sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820", "PR #6619", "PR #6680"] @@ -29,6 +29,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | pipeline `sampling_constraints`、`StageConfig.to_omegaconf()`、runtime stage config | `stage-config-propagation`：`VOMNI-CFG-1o` | `merge_pipeline_deploy` → `StageConfig.to_omegaconf()` → `engine.stage_configs` | | stage transport、`requires_full_payload_input`、topology projection 或 override rejection | `stage-transport`：`VOMNI-CFG-1p` | `stage_config.py::{StagePipelineConfig,_build_engine_args}` → `omni_config.py::_build_model_config` → `engine/arg_utils.py::OmniEngineArgs` | | HF cache snapshot path、空 `config.json`、name-based pipeline fallback | [`model-reference-routing`](rules-model-reference-routing.md#conf-7a-模型引用解析必须物化对象存储配置并只从受控名称组件匹配)：`CONF-7a` | `config_factory.py::{_name_match_candidate,StageConfigFactory._try_infer_model_type}` → `pipeline_registry.py::OMNI_PIPELINES` → `tests/config/test_config_factory.py::TestNameMatchCandidateSnapshotPaths` | +| Qwen3-TTS adaptive codec chunk keys、fixed-ramp precedence | [Qwen3-TTS `Q3TTS-3e`](../../models/qwen3-tts/rules.md#q3tts-3e-adaptive-ramp-是-host-side每段-opt-in-控制器不是-cuda-graph-计划) | `deploy/qwen3_tts{,_high_concurrency}.yaml` connector `extra` → `stage_input_processors/chunk_size_utils.py::parse_adaptive_config` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8c5373599..41939aae1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ da94c3eb` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 4ad455e6` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 62ba3e4ca..af6e615e1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/async-chunk.md b/knowledge/repos/vllm-omni/components/distributed/async-chunk.md index d4ae2b747..573df8af0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/async-chunk.md +++ b/knowledge/repos/vllm-omni/components/distributed/async-chunk.md @@ -1,7 +1,7 @@ --- title: "Async chunk（跨 stage 分块流式）" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-09-05 type: guide tags: [vllm-omni, components, distributed] sources: [docs/design/feature/async_chunk.md, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py] @@ -22,7 +22,10 @@ sources: [docs/design/feature/async_chunk.md, vllm_omni/distributed/omni_connect Thinker→Talker 逐 decode step（通常 1）；Talker→Code2Wav 积累到 `codec_chunk_frames`（默认 25）再发——初始阶段按服务负载动态选初始 chunk（IC） 以降 TTFP，可用逐请求 API 字段 `initial_codec_chunk_frames` 覆盖；Code2Wav 流式 - 解码（支持批推理）。 + 解码（支持批推理）。Qwen3-TTS 可 opt-in `codec_chunk_adaptive`：chunk 0 仍使用 + dynamic IC，chunk 1+ 由 host-side、每 request/segment 的 EWMA buffer-feedback 控制； + fixed `codec_chunk_ramp` 在此模式不生效。segment finish 与 request cleanup 都必须丢弃 + controller state，不能把上一段的 buffer/telemetry 继承到下一段。^[PR #6001] - 配置：deploy 顶层 `async_chunk`（默认 true）；端到端跑完的 pipeline 应 pin `false`——单 stage diffusion 必须 `async_chunk: false` 的事故见 [ci-gotchas](../../ci/guides/ci-gotchas.md) 第 2 条。 diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index ef64ca661..1b867cfa4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -1,10 +1,10 @@ --- title: "Distributed 传输规则" created: 2026-08-05 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", "PR #6089", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406"] +sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406"] confidence: high --- @@ -35,6 +35,12 @@ confidence: high - 验收：覆盖非空→空→非空、upstream exhaustion、stage completion、重复 terminal update 和窗口恢复；首批测试看 `test_chunk_transfer_adapter.py`。 +## DIST-1b1 — per-segment adaptive controller 不得越过 connector lifecycle + +- 触发：Qwen3-TTS adaptive chunk、resumable segment finish，或 transfer adapter/local runner cleanup 修改。 +- 强制：controller 可按 request 持有，但 `is_segment_finished` 与 request cleanup 均必须清除；下一段从 chunk 0 重新建立 EWMA、emitted ledger、playback cursor 与 telemetry。request-global connector continuity key 不能延长 controller 生命周期。 +- 验收：normal finish、segment finish、abort cleanup 与 empty terminal payload 都断言 state 不残留；覆盖 `OmniChunkTransferAdapter` 和 `OmniConnectorModelRunnerMixin`。^[PR #6001] + ## DIST-1c — payload connector 按 edge 所有权创建且不与 KV manager 捆绑 - 触发：修改 AR/DiT 解耦、stage connector role、`custom_process_next_stage_input_func`、 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 679979532..16c3a390e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index f644903b6..a46cc9d47 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -1,7 +1,7 @@ --- title: "Model Executor 规则" created: 2026-07-10 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149"] @@ -29,6 +29,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | Talker-MTP FULL graph、平台 capture 能力、显式 opt-out | `mtp-graph`：`EXEC-4c` | `platforms/interface.py::supports_talker_mtp_graph_capture` → platform override → model `talker_mtp_graph_safe` → `OmniGPUModelRunner._init_talker_mtp` | | `model_extras`、shared T2I/I2V example、canonical prompt envelope | `image-task-envelope`：`EXEC-6a` | `examples/offline_inference/{text_to_image/text_to_image.py,image_to_video/image_to_video.py}` → `model_extras/registry.py::{build_text_to_image_prompt,build_image_to_video_prompt}` → model pipeline validation | | CosyVoice3 non-module TensorRT CFM、stream/event ownership、raw-pointer buffer lifetime 或 context-pool reuse | [CosyVoice3 TensorRT handoff](../../models/cosyvoice3/rules.md)：`COSYVOICE3-1a` | `code2wav_core/cfm.py::ConditionalCFM.forward_estimator` → `flow_estimator_trt.py::TrtContextWrapper` | +| Qwen3-TTS adaptive chunk、EWMA target、all-frame emit、controller cleanup | [Qwen3-TTS adaptive ramp](../../models/qwen3-tts/rules.md#q3tts-3e-adaptive-ramp-是-host-side每段-opt-in-控制器不是-cuda-graph-计划) | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `chunk_size_utils.py::{AdaptiveChunkController,compute_adaptive_emit}` → Code2Wav | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index de02238ed..3d0ade5d9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fb1e4d7c0..f9d1915d5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ da94c3ebe15c78f5c580936f6b18a6d97d03539b` 验证存在 +- 源码校验：以上路径均已在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index e5cbdb94e..2fe926d65 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ da94c3eb`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4ad455e6`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -128,7 +128,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 13173d6b \ - --to da94c3eb \ + --to 4ad455e6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md index 97cf879a4..28aafcb08 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md @@ -1,10 +1,10 @@ --- title: "Qwen3-TTS" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-05 type: index tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py] +sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py] confidence: high --- @@ -13,7 +13,7 @@ confidence: high ## 名称与范围 - 正式 owner：Qwen3-TTS serving、ref audio、artifact cache、async Code2Wav - 增量解码/状态生命周期与模型专有 NPU patch。事实在 `main @ 4f9b6507` + 增量解码/状态生命周期、adaptive chunk ramp 与模型专有 NPU patch。事实在 `main @ 4ad455e6` 复核。 - serving 入口：`vllm_omni/entrypoints/openai/serving_speech.py`。 - NPU tokenizer patch：`vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py`；共享 diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index 910b0214d..9c173b66e 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -1,10 +1,10 @@ --- title: "Qwen3-TTS 规则" created: 2026-07-20 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6523", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] +sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6523", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] confidence: high --- @@ -18,7 +18,7 @@ confidence: high |---|---|---| | Qwen3-TTS、`qwen3_tts` pipeline | Q3TTS-1a/1b | `config/pipeline_registry.py::OMNI_PIPELINES["qwen3_tts"]`；`model_executor/models/qwen3_tts/pipeline.py` | | `ref_audio`、x-vector、ICL、artifact-only reuse | Q3TTS-1a/1b/1c | `entrypoints/openai/serving_speech.py::_qwen3_tts_can_use_ref_audio_artifact_only`、`_track_ref_audio_artifact_warmup`、`_mark_ref_audio_artifact_ready_for_request` | -| talker/code2wav、delta frame、request cache、segmented graph | Q3TTS-3a/3b/3c/3d + Model Executor | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `qwen3_tts_code2wav.py::Qwen3TTSCode2Wav` → `segmented_graph_wrapper.py` | +| talker/code2wav、adaptive chunk、delta frame、request cache、segmented graph | Q3TTS-3a/3b/3c/3d/3e + Model Executor | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `stage_input_processors/chunk_size_utils.py` → `qwen3_tts_code2wav.py::Qwen3TTSCode2Wav` → `segmented_graph_wrapper.py` | | OpenAI speech adapter | Q3TTS-1a/1b + Serving | `entrypoints/openai/tts_adapters/qwen3_tts.py::Qwen3TTSAdapter` → `serving_speech.py` | | NPU、RoPE、BNSD/BSND、`codec_chunk_ramp` | Q3TTS-2a | `platforms/npu/models/qwen3_tts_tokenizer_v2.py::_apply_rotary_pos_emb_npu` → `platforms/npu/layers/rotary_embedding.py::npu_rotary_mul_with_bsnd_fallback` | @@ -140,6 +140,19 @@ confidence: high 最终 pin 没有 PR body 早期描述的 `_last_output_audio_length`；真实合同是 exact-length `list[Tensor]` 加 Code2Wav 的 stateless leading-context trim。 +## Q3TTS-3e — adaptive ramp 是 host-side、每段 opt-in 控制器，不是 CUDA graph 计划 + +- 触发：`codec_chunk_adaptive`、Talker→Code2Wav async chunk、动态 IC、buffer/underrun telemetry，或 Qwen3-TTS deploy 的 chunk key。 +- 强制：默认关闭；启用后优先于 fixed `codec_chunk_ramp`。chunk 0 保持 dynamic IC（即使配置了 fixed initial IC），chunk 1+ 才逐 emit 从本段 controller 决定 target。controller 以 80 ms/frame 的 emitted audio 减去 first-emit 后 wall time 得 buffer；以 EWMA（alpha 0.3）估算 frame time。greedy target、无条件 ramp floor `last_target + max(ramp_delta_min, int(ewma/ramp_divisor))` 和 `codec_chunk_min_frames..codec_chunk_frames` clamp 合并；buffer 达到 `codec_chunk_frames * ewma + safety_margin` 后 hysteresis lock 在 max。floor 在负 buffer 下也不能关闭：即使 W5 streaming decoder state 已在基线，per-chunk transport/padding/graph overhead 与 overload death-spiral 仍使向下缩块不是本 PR 的策略；downward adaptation 明确 deferred。 +- 强制：非终止也发送所有已积累的 new frames；finished flush 全部余帧，无余帧仍发 empty-finished sentinel。recorded `last_target` 必须是本次 intended target，不能用实际 overshoot emitted frames，否则 spike 会把 floor 错误 ratchet 到 max。controller state 在 segment boundary/request cleanup 清除；每段 INFO telemetry 记录 chunk trajectory、每块 gap 与 total gap。 +- 配置：`codec_chunk_adaptive=false`、`codec_chunk_min_frames=2`、`codec_chunk_safety_margin_ms=50.0`、`codec_chunk_ramp_divisor=15`、`codec_chunk_ramp_delta_min=2`；min/divisor/delta-min 小于 1 clamp 为 1，最大值固定是 `codec_chunk_frames`（没有单独 adaptive max key）。两份 Qwen3-TTS deploy 只注释示例，未默认开启。 +- CUDA 边界（未解决、非阻塞）：CUDA 的 Code2Wav graph capture grid 从 fixed IC/ramp 建，而 controller runtime cumulative boundaries 通常不在该 grid；adaptive ramp（含 dynamic-IC chunk 0）因此可 eager，910B NPU 不受该 CUDA graph path 影响。已匹配 source 的 nonterminal partial replay 还可能不 advance decoder cache。这两点尚无 merged fix，不得声称 adaptive 与 fixed ramp 同等 graph replay 或 cache continuity；在 CUDA 上以 graph stats/hit rate 实测。^[PR #6001] + +### PR #6001 性能证据边界 + +- 这不是通用吞吐/RTF 改善承诺。合并前 A/B 是 Ascend 910B、vLLM 0.27.1 + vLLM-Ascend PR #14027、Qwen3-TTS voice-clone、40 requests、concurrency 8；adaptive 为 opt-in 唯一变量：mean RTF 0.746（default 0.757 / fixed 0.789）、mean underrun 0.14 s（1.18 / 0.27），continuity OK 35%（0% / 22.5%）。另一 910B c=2 观测 RTF 0.56、underrun 0.02 s；c=8 per-stream RTF>1 仍会饱和。作者听音验证质量，但未给跨硬件或 CI unit suite 证据。 +- 更准确的主张是 portability：约 11 ms/frame 的 CUDA hardware 不重调 Ascend 手选 `[4,4,8,16,25]` 即自适应出 `[2,4,25,25,3]`；CUDA graph miss 可解释相对 fixed ramp 的损失。上述数字、模型、负载和软件栈以外不得外推。^[PR #6001] + ## PR #5202 性能证据边界 - 作者基准是单张 RTX 3090 24 GB、vLLM 0.26.0、Omni From 6e3e57acd723a48732b683ebd5a7a6d5e10edbfd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 00:54:27 +0800 Subject: [PATCH 349/465] knowledge(vllm-omni): sync d5aced92 (PR #6717) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +-- .../diffusion/rules-upstream-compat.md | 10 ++++++++-- .../diffusion/rules-worker-observability.md | 18 ------------------ .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 4 ++-- 11 files changed, 19 insertions(+), 32 deletions(-) delete mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b700c088c..db06e2cae 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: da94c3ebe15c78f5c580936f6b18a6d97d03539b + previous_audited_sha: 4ad455e61e2857450594189895fe860596f402dc audited_ref: main - audited_sha: 4ad455e61e2857450594189895fe860596f402dc + audited_sha: d5aced92966e6f12134fe9a30679fdeab07e4bb9 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8659b9e37..38a286341 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4ad455e61e2857450594189895fe860596f402dc`. The canonical machine baseline +d5aced92966e6f12134fe9a30679fdeab07e4bb9`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7762afd0b..98a96de6a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 41939aae1..e528ed8f9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4ad455e6` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ d5aced92` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -31,7 +31,6 @@ sources: [] | 理解共享职责和数据流 | [architecture](architecture.md) | | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | image/diffusion timing、step normalization、missing-versus-zero 与 benchmark 解释边界 | [metrics evidence rules](rules-metrics.md) | -| worker process title、topology rank 与日志前缀 | [worker observability rules](rules-worker-observability.md) | | vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md index f9fba64a8..465ef1576 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md @@ -1,10 +1,10 @@ --- title: "Diffusion upstream 兼容规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py, "PR #6287", "PR #6273"] +sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py, "PR #6287", "PR #6273", "PR #6307", "PR #6717"] confidence: high --- @@ -46,3 +46,9 @@ confidence: high - 禁止：直接从 `triton` 或 `triton.language` 导入，或把导入检查通过误认为真实 kernel 已完成跨平台验证。 - 验收：静态检查确认无直接 Triton 导入，并在支持的 CUDA 环境执行目标 kernel 的导入与定向回归；非 CUDA fallback 仍需独立验证。^[PR #6273] +## DIFF-4t — 重新引入拓扑可观测性前必须对齐 live parallel-state API + +- 触发：为 diffusion worker 添加或恢复基于并行拓扑的进程标题、日志前缀或 rank 标签。 +- 强制：从目标基线实际存在的 parallel-state API 读取 topology；在重排/rebase 后重新审查全部 group getter 与初始化时序，并以目标基线的定向测试和 CI 通过作为合入证据。 +- 禁止：把已撤回的 topology-aware title 当作当前行为，或复用已被 parallel-state 布局变更淘汰的 getter。 +- 验收：实现引用均能在当前基线解析，覆盖初始化前后边界并通过 post-rebase CI。^[PR #6307] ^[PR #6717] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md b/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md deleted file mode 100644 index 90b10069d..000000000 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: "Diffusion worker observability rules" -created: 2026-09-05 -updated: 2026-09-05 -type: rule -tags: [vllm-omni, components, diffusion] -sources: ["PR #6307", vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_diffusion_worker_process_title.py] -confidence: high ---- - -# Diffusion worker observability rules - -## DIFF-4o — worker 进程标题和日志前缀必须来自同一已初始化拓扑 - -- 触发：修改 `DiffusionWorker`/`WorkerProc` 启动、model-parallel 初始化，或 worker 的进程标题和日志装饰。 -- 强制：early startup 先用通用 `DiffusionWorker` 标识；`initialize_model_parallel()` 完成后、模型加载前再次从 active group 生成同一个标识并同时交给 upstream `set_process_title(..., prefix="vLLM-Omni")` 与 `decorate_logs()`。按 DP、PP、SP、CFG、TP、FS（HSDP）顺序附加非 singleton group 的 `_`；仅在 expert parallel enabled 时查询 EP 并附加它。所有 singleton 维度省略。 -- 禁止：用 global rank 代替局部 topology rank；在 parallel state 可用前读取 group；仅更新 OS title 或仅更新日志前缀；因 optional `setproctitle` 缺失让 worker 启动失败；把该可观测性改动描述为 scheduling、通信、模型输出或性能路径变化。 -- 验收：覆盖初始化前 generic 名称及不读取 group、singleton、DP/PP/SP/CFG/TP、standalone FS/HSDP、conditional EP、缺少 `setproctitle` 仍装饰日志，以及 Linux `ps` 可见 title；冷启动 `ps` polling 要有足够 wall-clock budget，避免 import 延迟造成 flaky test。此状态的 `get_fs_group` 依赖在随后 parallel-state 改版中会不兼容，属于本提交之后的边界，不能回写为本提交未实现。^[PR #6307] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index af6e615e1..30233693e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 16c3a390e..49a1a9688 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3d0ade5d9..b7749bcfb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f9d1915d5..df82716f0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4ad455e61e2857450594189895fe860596f402dc` 验证存在 +- 源码校验：以上路径均已在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2fe926d65..8cbbbd7db 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4ad455e6`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d5aced92`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -128,7 +128,7 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ --from 13173d6b \ - --to 4ad455e6 \ + --to d5aced92 \ --repo \ --mode report-only ``` From 660b58c4f2c017c6c278abb723334dd3b01fba5f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:03:27 +0800 Subject: [PATCH 350/465] knowledge(vllm-omni): sync be335a86 (PR #6664) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 9 ++++++--- .../repos/vllm-omni/models/moss-tts/_index.md | 6 +++--- .../vllm-omni/models/moss-tts/architecture.md | 16 +++++++++++++--- 11 files changed, 31 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index db06e2cae..24dc3df3d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4ad455e61e2857450594189895fe860596f402dc + previous_audited_sha: d5aced92966e6f12134fe9a30679fdeab07e4bb9 audited_ref: main - audited_sha: d5aced92966e6f12134fe9a30679fdeab07e4bb9 + audited_sha: be335a86f9af5bde108e11fb3e01d729553a3532 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 38a286341..80081fe82 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d5aced92966e6f12134fe9a30679fdeab07e4bb9`. The canonical machine baseline +be335a86f9af5bde108e11fb3e01d729553a3532`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 98a96de6a..818ea4108 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ be335a86` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e528ed8f9..8104e98b2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d5aced92` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ be335a86` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 30233693e..dcfb6738b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ be335a86` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 49a1a9688..1404025d6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ be335a86` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b7749bcfb..006c98026 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ be335a86` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index df82716f0..f6c548100 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d5aced92966e6f12134fe9a30679fdeab07e4bb9` 验证存在 +- 源码校验：以上路径均已在 `main @ be335a86` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8cbbbd7db..48244e278 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d5aced92`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ be335a86`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -53,6 +53,9 @@ PR #6634 adds Omni-DuplexEval benchmark support without changing the AR registry registry, `OMNI_PIPELINES`, or deploy-YAML inventory. Its MiniCPM-o duplex endpoint is an evaluation consumer, not a new model registration or deploy profile. ^[PR #6634] +PR #6664 only extends BF16 autocast for MossAudioTokenizer v2 decode on NPU; it does not +change any AR/diffusion registry, `OMNI_PIPELINES`, or deploy-YAML inventory. ^[PR #6664] + 对比上一审计快照（`39c16d75`）：AR 架构 88→81，diffusion pipeline 65→64， OMNI_PIPELINES 57→55，deploy 86→84。PR #6353 删除 AudioX 的 diffusion registry、 processor/extras/recipe/examples/tests，并删除 MammothModa2 Preview/Dev 共用的七个 AR @@ -127,8 +130,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 13173d6b \ - --to d5aced92 \ + --from d5aced92 \ + --to be335a86 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/moss-tts/_index.md b/knowledge/repos/vllm-omni/models/moss-tts/_index.md index fe71f5d11..c8d08ad80 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/_index.md @@ -1,15 +1,15 @@ --- title: "MOSS-TTS 家族（Delay/Realtime/Local/Nano,一族八 deploy）" created: 2026-07-21 -updated: 2026-09-02 +updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: ["PR #5635", vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/model_executor/models/moss_tts/, vllm_omni/model_executor/models/moss_tts_nano/, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/] +sources: ["PR #5635", "PR #6664", vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/model_executor/models/moss_tts/, vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py, vllm_omni/model_executor/models/moss_tts_nano/, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/] --- # MOSS-TTS 家族 -以下事实在 `main @ 740cb35a` 复核。 +以下事实在 `main @ be335a86` 复核。 ## 名称与范围 diff --git a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md index 66b4125ed..ceb641c4a 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md @@ -1,15 +1,15 @@ --- title: "MOSS-TTS 架构" created: 2026-07-21 -updated: 2026-09-02 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models] -sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, vllm_omni/model_executor/models/moss_tts/pipeline.py, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/moss_voice_generator.yaml] +sources: ["PR #5635", "PR #5235", "PR #6664", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, vllm_omni/model_executor/models/moss_tts/pipeline.py, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/moss_voice_generator.yaml] --- # MOSS-TTS 架构 -事实在 `main @ 740cb35a` 复核;变体/deploy 速查见 [index](_index.md)。 +事实在 `main @ be335a86` 复核;变体/deploy 速查见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 @@ -27,6 +27,11 @@ sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts wrapper（批解码 vs 流式解码）;按 raw codec config 的 `number_channels` 运行期选代， 字段缺失/单通道走 v1，≥2 走 v2。projection 的 Linear/Identity topology 同样属于 checkpoint ABI，具体门禁见 [rules](rules.md)。 +- v2 decode 的 dtype 边界：#5235 的 decode-LUT 故意使用 `.float()` 保留精度，故 + quantizer 可产出 FP32 embedding；它们随后经过 80 层 decoder transformer 的 BF16 + LayerNorm。CUDA 已以包住完整 quantizer→decoder 段的 BF16 autocast 处理它；#6664 + 将同一 autocast 扩展给 NPU，避免 `aclnnLayerNorm` 拒绝 FP32 input/BF16 weight。 + CPU 仍不启用 autocast；这不是 codec 算法或音质变化，也不能外推其他平台支持。 - 共享框架面：[Config 组件](../../components/configuration/architecture.md)、 SharedMemoryConnector、speaker cache（`reference_encoder.py`,树内注明援引 Fish/CosyVoice3/Qwen3-TTS 先例）。 @@ -70,3 +75,8 @@ sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts - 已知未决：delay 路下 connector 级 `codec_chunk_frames` 是否还有残余作用 未追;`moss_tts_local_codec` 与 `moss_tts_codec` 在 serving 侧的分流细节 未逐行读。v1/v2 选择与 projection topology 已复核，但 pin 上没有对应自动化 regression。 + +NPU dtype 修复的上游运行记录仅限 Ascend 910B、vLLM v0.27.1、 +`vllm-ascend b4b04c5eb` 与 Moss-TTS-Local-v1.5 的 serve/generate 成功。复验必须在 +此硬件/软件/模型组合真实走 codec decode；该 smoke 仅证明此 mixed-dtype crash 被解除， +不能据此声明音频质量或泛平台兼容性。 From 9e739c70960cbed416de3272f6617e88141e83ce Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:08:56 +0800 Subject: [PATCH 351/465] knowledge(vllm-omni): sync ebb3b205 (PR #6390) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 11 +++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/flux2/_index.md | 6 +++--- .../repos/vllm-omni/models/flux2/rules.md | 21 +++++++++++++++++-- 12 files changed, 44 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 24dc3df3d..0ff630135 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d5aced92966e6f12134fe9a30679fdeab07e4bb9 + previous_audited_sha: be335a86f9af5bde108e11fb3e01d729553a3532 audited_ref: main - audited_sha: be335a86f9af5bde108e11fb3e01d729553a3532 + audited_sha: ebb3b20576736ec3ad264788b8721ec61b28aa8d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 80081fe82..591d195bc 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -be335a86f9af5bde108e11fb3e01d729553a3532`. The canonical machine baseline +ebb3b20576736ec3ad264788b8721ec61b28aa8d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 4afdf939a..4d94370e1 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704"] --- # vLLM-Omni CI @@ -24,6 +24,15 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie - PR #6404 adds CPU/unit coverage for closing-append frame placement, held last/still frames, two-frame stacking, Stage 0 marker/embedding ordering, rejected early video appends, dtype conversion, and external WAV video demos. It is protocol and input-assembly coverage; its unchecked full-model video E2E and manual demo do not establish an end-to-end checkpoint pass or video quality result. ^[PR #6404] +## FLUX.2 configurable encoder-layer test boundary + +- `tests/model_tests/diffusion/diff_model_builders.py::tiny_flux2_builder` makes the fixture + economical by reducing the text encoder to three layers and selecting `[0, 1, 2]`; its + transformer `joint_attention_dim=96` matches the concatenated selected hidden outputs. + The focused offline `Flux2Pipeline` run covers text-to-image, determinism, and multi-output. + It does not execute a full-size legacy checkpoint without `text_encoder_out_layers`, so it is + not evidence that the `(10, 20, 30)` fallback completed a full-model validation. ^[PR #6390] + ## 不放什么 - 跨仓库通用的测试方法。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 818ea4108..3b11f369e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ be335a86` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ebb3b205` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8104e98b2..926840d77 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ be335a86` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ ebb3b205` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index dcfb6738b..66647845c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ be335a86` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ebb3b205` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1404025d6..b6de3fc6c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ be335a86` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ebb3b205` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 006c98026..54bf5da49 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ be335a86` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ebb3b205` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f6c548100..efc887cad 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ be335a86` 验证存在 +- 源码校验：以上路径均已在 `main @ ebb3b205` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 48244e278..f104180e0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ be335a86`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ebb3b205`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -130,8 +130,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from d5aced92 \ - --to be335a86 \ + --from be335a86 \ + --to ebb3b205 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/flux2/_index.md b/knowledge/repos/vllm-omni/models/flux2/_index.md index 4d623c41c..4fd58ac86 100644 --- a/knowledge/repos/vllm-omni/models/flux2/_index.md +++ b/knowledge/repos/vllm-omni/models/flux2/_index.md @@ -1,10 +1,10 @@ --- title: "FLUX.2" created: 2026-07-20 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5136", "PR #3027", vllm_omni/diffusion/models/flux2/, vllm_omni/diffusion/models/mistral_encoder/] +sources: ["PR #5136", "PR #3027", "PR #6390", vllm_omni/diffusion/models/flux2/, vllm_omni/diffusion/models/mistral_encoder/] confidence: high --- @@ -19,6 +19,6 @@ confidence: high ## 什么时候查这里 -- 审查 FLUX.2 text-encoder-only FP8、transformer block 的 online FP8 prefix、component prefix、meta 初始化或 CPU offload。 +- 审查 FLUX.2 text-encoder-only FP8、transformer block 的 online FP8 prefix、component prefix、meta 初始化或 CPU offload，或 Mistral hidden-state 提取层的 checkpoint 配置。 - 描述直达源码与具体不变量见 [FLUX.2 rules](rules.md#direct-代码快速入口)；通用模型适配检查见 [model adaptation guardrails](../../review/guides/model-adaptation-guardrails.md)。 diff --git a/knowledge/repos/vllm-omni/models/flux2/rules.md b/knowledge/repos/vllm-omni/models/flux2/rules.md index 382069cd5..c443ca7e0 100644 --- a/knowledge/repos/vllm-omni/models/flux2/rules.md +++ b/knowledge/repos/vllm-omni/models/flux2/rules.md @@ -1,10 +1,10 @@ --- title: "FLUX.2 规则" created: 2026-07-20 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4645", "PR #5136", vllm_omni/diffusion/models/flux2/flux2_transformer.py, "PR #5910", "PR #3027"] +sources: ["PR #4645", "PR #5136", vllm_omni/diffusion/models/flux2/flux2_transformer.py, "PR #5910", "PR #3027", "PR #6390", vllm_omni/diffusion/models/flux2/pipeline_flux2.py, tests/model_tests/diffusion/diff_model_builders.py] confidence: high --- @@ -20,6 +20,7 @@ confidence: high | Mistral text encoder、FP8、component namespace | FLUX2-1a | `diffusion/models/mistral_encoder/` → component quantization selector/loader | | meta parameter、CPU offload、BF16 baseline | FLUX2-1b | FLUX.2 component loader/materialization 路径 → `Flux2Pipeline` | | SP auto-padding、`mask_sp_padding`、dense/varlen | [SPPAD-1a](../../components/diffusion/rules-sp-padding.md#sppad-1a-padding-mask-是显式正确性性能策略不是无损优化) | Flux2 `_sp_plan` → forward context → `hidden_states_mask` → attention | +| text encoder hidden-state 层、`text_encoder_out_layers`、prompt embedding 宽度 | FLUX2-1d | `pipeline_flux2.py::Flux2Pipeline.__init__` → `encode_prompt` → `__call__` → Mistral encoder | FLUX.1 和 FLUX.2-Klein 不自动归到本页；必须由描述/registry key 明确命中对应 owner。 @@ -56,5 +57,21 @@ FLUX.1 和 FLUX.2-Klein 不自动归到本页；必须由描述/registry key 明 - 验收：用 dynamic `Fp8Config` 构造小模型，分别枚举两类 block 的 config 与 prefix，并断言 三个 outer projection 仍是 `nn.Linear`。 ^[PR #3027] +## FLUX2-1d — 选择的 Mistral hidden states 是 checkpoint 属性，不是请求参数 + +- 触发：修改 `Flux2Pipeline` 的 prompt encoding、`text_encoder_out_layers`、Mistral config，或 + FLUX.2 tiny-model builder 的 encoder/transformer geometry。 +- 强制：构造期从 `text_encoder.config.text_encoder_out_layers` 读取并固化为 pipeline 实例的 + tuple；旧 checkpoint 没有该字段时只回退到 `(10, 20, 30)`。`encode_prompt` 及正/负 prompt 的 + `__call__` 路径都必须使用同一实例值，把选出的 hidden outputs 按既有顺序拼接，并保持结果宽度与 + transformer `joint_attention_dim` 一致。 +- 禁止：从 request `extra_args` 或 `encode_prompt`/`__call__` 参数在推理时覆盖层集合；模型是在 + 特定抽取层上训练的，Diffusers 的可传参数不改变这个 FLUX.2 checkpoint contract。也不得只缩小 + text encoder 层数却不同时缩小选择列表或 `joint_attention_dim`。 +- 验收：tiny builder 使用 3 个 text-encoder layers、`[0, 1, 2]` 和 + `joint_attention_dim=96`；离线 `Flux2Pipeline` 的 text-to-image、determinism 和 multi-output + 三个子测试通过。该 PR 未运行“无 config 字段”的完整尺寸 checkpoint，故 legacy fallback 的 + full-size 行为仍是源码合同，不应写成已完成的 full-model 验证。 ^[PR #6390] + 共享 component-quantization 合同见 [Diffusion rules](../../components/diffusion/rules.md)； 加载器 upstream 边界见 [loading drift](../../rebase/upstream-api-drift-loading.md)。 From 06fb5d650513f5d99cbbe62108fc812a496c4abf Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:15:00 +0800 Subject: [PATCH 352/465] knowledge(vllm-omni): sync 6d84409a (PR #5464) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 15 ++++++++++++++- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 26 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0ff630135..c86e35132 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: be335a86f9af5bde108e11fb3e01d729553a3532 + previous_audited_sha: ebb3b20576736ec3ad264788b8721ec61b28aa8d audited_ref: main - audited_sha: ebb3b20576736ec3ad264788b8721ec61b28aa8d + audited_sha: 6d84409a683aeef476ad2d47a839998ef2344b4b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 591d195bc..80b08220b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ebb3b20576736ec3ad264788b8721ec61b28aa8d`. The canonical machine baseline +6d84409a683aeef476ad2d47a839998ef2344b4b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 4d94370e1..028dbd252 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464"] --- # vLLM-Omni CI @@ -13,6 +13,19 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie - 处理 vLLM-Omni 的 L2/L4、模型测试配置或 CI 特有问题。 +## MiniCPM-o 4.5 non-async transfer regression boundary + +- The target-state tests add two CPU/unit regressions only: disabling `async_chunk` during + deploy/pipeline merging must select the Code2Wav full-payload producer, and a `None` + full payload must produce an empty terminal payload (`last_chunk` and `finished`) so the + consumer wait gate can release. The pre-existing topology test already checks that the + pipeline declares full-payload, async-chunk, and token-only hooks; PR prose is not evidence + that this commit added that hook-declaration coverage. +- These assertions do not modify or execute the runtime implementation, offline inference, + online serving, connector transport, or an end-to-end MiniCPM-o checkpoint. They are scoped + selection/terminal-metadata regression coverage, and were reverted immediately by PR #6730; + retain them only for the `6d84409` target-state audit. ^[PR #5464] ^[PR #6730] + ## MiniCPM native-duplex server-VAD coverage - Ready/merge 的 MiniCPM duplex source dependencies 纳入 server-VAD runner；其 L4 full-model diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3b11f369e..3cf3f4d15 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ebb3b205` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6d84409a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 926840d77..28f612e0b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ebb3b205` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 6d84409a` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 66647845c..e7e109ef1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ebb3b205` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6d84409a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b6de3fc6c..7758928be 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ebb3b205` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6d84409a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 54bf5da49..11f203f4c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ebb3b205` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6d84409a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index efc887cad..f5d413849 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ebb3b205` 验证存在 +- 源码校验：以上路径均已在 `main @ 6d84409a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f104180e0..c41928ada 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ebb3b205`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6d84409a`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -130,8 +130,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from be335a86 \ - --to ebb3b205 \ + --from ebb3b205 \ + --to 6d84409a \ --repo \ --mode report-only ``` From 315152e6f0f09a8e2de2ec0d754e94b3e52fcf8a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:19:42 +0800 Subject: [PATCH 353/465] knowledge(vllm-omni): sync 6b9ce05f (PR #6730) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 20 +++++++------------ .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 19 insertions(+), 25 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c86e35132..d3430cf22 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ebb3b20576736ec3ad264788b8721ec61b28aa8d + previous_audited_sha: 6d84409a683aeef476ad2d47a839998ef2344b4b audited_ref: main - audited_sha: 6d84409a683aeef476ad2d47a839998ef2344b4b + audited_sha: 6b9ce05febbb9ed1431c1570e967c6a2a85ff942 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 80b08220b..cab4698fe 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6d84409a683aeef476ad2d47a839998ef2344b4b`. The canonical machine baseline +6b9ce05febbb9ed1431c1570e967c6a2a85ff942`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 028dbd252..7f25d9963 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730"] --- # vLLM-Omni CI @@ -13,18 +13,12 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie - 处理 vLLM-Omni 的 L2/L4、模型测试配置或 CI 特有问题。 -## MiniCPM-o 4.5 non-async transfer regression boundary - -- The target-state tests add two CPU/unit regressions only: disabling `async_chunk` during - deploy/pipeline merging must select the Code2Wav full-payload producer, and a `None` - full payload must produce an empty terminal payload (`last_chunk` and `finished`) so the - consumer wait gate can release. The pre-existing topology test already checks that the - pipeline declares full-payload, async-chunk, and token-only hooks; PR prose is not evidence - that this commit added that hook-declaration coverage. -- These assertions do not modify or execute the runtime implementation, offline inference, - online serving, connector transport, or an end-to-end MiniCPM-o checkpoint. They are scoped - selection/terminal-metadata regression coverage, and were reverted immediately by PR #6730; - retain them only for the `6d84409` target-state audit. ^[PR #5464] ^[PR #6730] +## Historical MiniCPM-o 4.5 unit-test revert + +- PR #6730 reverted the two PR #5464 CPU assertions after CI failure. They are not current + coverage and do not evidence a runtime, offline-inference, online-serving, or transport + implementation change. Reintroduce comparable coverage only with a passing CI-backed change. + ^[PR #5464] ^[PR #6730] ## MiniCPM native-duplex server-VAD coverage diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3cf3f4d15..65cb85c9d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6d84409a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6b9ce05f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 28f612e0b..9a2a6b3c5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6d84409a` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 6b9ce05f` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e7e109ef1..cdb14a94f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6d84409a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6b9ce05f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7758928be..33b62b7e2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6d84409a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6b9ce05f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 11f203f4c..75d21b709 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6d84409a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6b9ce05f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f5d413849..fd404638d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6d84409a` 验证存在 +- 源码校验：以上路径均已在 `main @ 6b9ce05f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c41928ada..120175b48 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6d84409a`（2026-08-28）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6b9ce05f`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -130,8 +130,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from ebb3b205 \ - --to 6d84409a \ + --from 6d84409a \ + --to 6b9ce05f \ --repo \ --mode report-only ``` From 34314a24ba78bc5fdf898424391edc8bf6f3bc5a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:28:26 +0800 Subject: [PATCH 354/465] knowledge(vllm-omni): sync 8bfad23f (PR #6722) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 ++- .../diffusion/rules-checkpoint-loading.md | 10 +++++----- .../diffusion/rules-upstream-compat.md | 9 +-------- .../diffusion/rules-worker-observability.md | 18 ++++++++++++++++++ .../vllm-omni/components/diffusion/rules.md | 3 ++- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 13 files changed, 39 insertions(+), 26 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d3430cf22..c5113e5e4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6d84409a683aeef476ad2d47a839998ef2344b4b + previous_audited_sha: 6b9ce05febbb9ed1431c1570e967c6a2a85ff942 audited_ref: main - audited_sha: 6b9ce05febbb9ed1431c1570e967c6a2a85ff942 + audited_sha: 8bfad23f98c118914bfedbaa46370d576c0add7d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index cab4698fe..2b3c91f80 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6b9ce05febbb9ed1431c1570e967c6a2a85ff942`. The canonical machine baseline +8bfad23f98c118914bfedbaa46370d576c0add7d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 65cb85c9d..15fcd535e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6b9ce05f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8bfad23f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9a2a6b3c5..bf38e16ba 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6b9ce05f` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 8bfad23f` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -32,6 +32,7 @@ sources: [] | 根据 PR 描述直达 execution parity、checkpoint/artifact identity、quality evidence 或 system-runtime 异常清理规则组与第一批源码 | [rules 与代码地图](rules.md) | | image/diffusion timing、step normalization、missing-versus-zero 与 benchmark 解释边界 | [metrics evidence rules](rules-metrics.md) | | vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | +| worker process title、拓扑局部 rank 与日志前缀 | [worker observability rules](rules-worker-observability.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | | Wan VAE height/width spatial reshard、empty tail、attention extent | [Wan spatial-shard rules](rules-wan-spatial-shard.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index 73f78d015..b07dab720 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -1,10 +1,10 @@ --- title: "Checkpoint、加载与量化合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6162", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", "PR #6651", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, vllm_omni/quantization/svdquant_config.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #4061"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6162", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", "PR #6651", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, vllm_omni/quantization/svdquant_config.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #6722", "PR #4061"] confidence: high --- @@ -157,9 +157,9 @@ confidence: high ## DIFF-2u — diffusion 并行状态必须保持单一拓扑所有权并原子清理 - 触发：修改 diffusion `parallel_state`、HSDP/FSDP2 `DeviceMesh`、SP/CFG/PP group、VAE patch parallel 或并行状态初始化与销毁。 -- 强制：`RankGenerator` 只表达真实正交的 TP/SP/PP/CFG/DP 轴；HSDP 不得作为额外轴，FSDP2 `DeviceMesh` 独占 replicate/shard 进程组；VAE tile parallel 直接从 WORLD 选 rank。初始化前验证 distributed 已启动且状态为空，组创建失败时必须清理本次已创建的 diffusion 与 vLLM 组；销毁时同时清空 vLLM `_PP` 等引用。 -- 禁止：重新创建冗余 `_DIT` 或 `_FS` 组；把 HSDP shard 当作 `RankGenerator` 的乘法维度；让 HSDP 继续依赖已移除的 fully-shard accessor；清理后保留已销毁的 pipeline group 或允许残留部分状态阻塞下一次初始化。 -- 验收：覆盖 HSDP standalone、HSDP+SP/CFG、VAE WORLD group、mesh 维度与实际 WORLD 不一致、初始化中途异常、重复初始化和销毁后重初始化；断言失败后无残留组，vLLM `_PP` 被置空，MiniMax H3 的 VAE/pipeline caller 使用 `get_world_group().device_group`，并以真实多 rank smoke 验证 DeviceMesh 与 collective 拓扑。^[PR #5531] +- 强制：`RankGenerator` 只表达真实正交的 TP/SP/PP/CFG/DP 轴；HSDP 不得作为额外轴。仅当 `use_hsdp` 时，在 post-#5531 atomic initializer 中建立 fully-sharded FS（连续 rank run）与 replica RP（同 shard position）group，并公开其 local rank；先验证 shard degree 为正且整除 WORLD，再创建任一 group。VAE tile parallel 直接从 WORLD 选 rank；初始化失败必须清理本次 diffusion 与 vLLM 组，销毁时清空 FS/RP 及 vLLM `_PP` 等引用。 +- 禁止：把 HSDP shard 当作 `RankGenerator` 的乘法维度；在非 HSDP 路径创建或查询 FS/RP；用 global rank 取代 group-local rank；在校验失败前分配 communicator，或清理后保留已销毁的 pipeline/FS/RP state。 +- 验收：覆盖 HSDP standalone、HSDP+SP/CFG、非法 zero/non-divisible shard size 在任何 group 创建前失败、FS/RP construction 与 teardown、重复初始化和销毁后重初始化；断言失败后无残留组，vLLM `_PP` 被置空，并以真实 multi-rank CPU/Gloo smoke 验证 collective topology。^[PR #5531] ^[PR #6722] ## DIFF-2v — SANA-WM Stage-1 checkpoint 必须保持标准 Diffusers 布局 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md index 465ef1576..5eeedb7d0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-upstream-compat.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py, "PR #6287", "PR #6273", "PR #6307", "PR #6717"] +sources: ["PR #5976", vllm_omni/diffusion/compile.py, vllm_omni/diffusion/layers/fused_moe.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py, vllm_omni/quantization/_copy_missing_attrs.py, tests/diffusion/test_compile.py, tests/e2e/accuracy/test_qwen_image.py, "PR #6287", "PR #6273"] confidence: high --- @@ -45,10 +45,3 @@ confidence: high - 强制：diffusion 代码统一从 `vllm.triton_utils` 导入 `triton` 与 `tl`，并保持现有 kernel 装饰器和调用语义。 - 禁止：直接从 `triton` 或 `triton.language` 导入，或把导入检查通过误认为真实 kernel 已完成跨平台验证。 - 验收：静态检查确认无直接 Triton 导入，并在支持的 CUDA 环境执行目标 kernel 的导入与定向回归；非 CUDA fallback 仍需独立验证。^[PR #6273] - -## DIFF-4t — 重新引入拓扑可观测性前必须对齐 live parallel-state API - -- 触发：为 diffusion worker 添加或恢复基于并行拓扑的进程标题、日志前缀或 rank 标签。 -- 强制：从目标基线实际存在的 parallel-state API 读取 topology；在重排/rebase 后重新审查全部 group getter 与初始化时序，并以目标基线的定向测试和 CI 通过作为合入证据。 -- 禁止：把已撤回的 topology-aware title 当作当前行为，或复用已被 parallel-state 布局变更淘汰的 getter。 -- 验收：实现引用均能在当前基线解析，覆盖初始化前后边界并通过 post-rebase CI。^[PR #6307] ^[PR #6717] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md b/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md new file mode 100644 index 000000000..1bc4814a1 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-worker-observability.md @@ -0,0 +1,18 @@ +--- +title: "Diffusion worker observability rules" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #6722", vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/test_diffusion_worker_process_title.py, tests/diffusion/distributed/test_expert_parallel_layout.py] +confidence: high +--- + +# Diffusion worker observability rules + +## DIFF-4w — worker title 与日志前缀必须共享已初始化的拓扑标识 + +- 触发：修改 `DiffusionWorker`/`WorkerProc` 启动、model-parallel 初始化、进程标题或日志装饰。 +- 强制：early startup 保留通用 `DiffusionWorker`；`initialize_model_parallel()` 完成后、模型加载前，按 DP、PP、SP、CFG、TP、FS、RP、EP 顺序从 active group 的 `rank_in_group` 追加非 singleton 维度，并把同一名字同时交给 `set_process_title(..., prefix="vLLM-Omni")` 和 `decorate_logs()`。FS/RP 只在 HSDP 查询，RP 只在多 replica layout 查询；EP 只在 expert parallel enabled 时查询。 +- 禁止：在 parallel state 可用前读 group；用 global rank 代替 topology local rank；让非 HSDP/非 EP 路径访问 FS/RP/EP；只更新 OS title 或日志之一；因 optional `setproctitle` 不可用让 worker 启动失败。 +- 验收：覆盖初始化前 generic 名称且不读 group、singleton、省略规则、DP/PP/SP/CFG/TP、FS、multi-replica RP、conditional EP、缺少 `setproctitle` 仍装饰日志，以及 Linux `ps` 可见 title；本提交的验证边界为 unit/CPU Gloo，未提供本地 GPU 证据。^[PR #6722] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 88a9b9311..d91aa18a6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] confidence: high --- @@ -34,6 +34,7 @@ confidence: high | Wan VAE spatial shard、gather/trim/reshard、empty tail、attention extent | `distributed-vae`：`DIFF-3b` | `distributed/autoencoders/wan_spatial_shard.py` → patched decoder attention/conv → gather final frame | | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、masked cross-attention、预算与 admission | `system-runtime`：`DIFF-4a`–`4j`, `DIFF-4v` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | +| worker title、log prefix、DP/PP/SP/CFG/TP/FS/RP/EP local rank | [worker observability](rules-worker-observability.md)：`DIFF-4w` | `diffusion_worker.py::_setup_diffusion_worker_proc_title_and_log_prefix` → `distributed/parallel_state.py` group getters | | image/diffusion service time、forward time、missing-versus-zero、per-step metric | [`metrics-evidence`](rules-metrics.md)：`DIFF-5a`, `DIFF-5b` | output formatter/sampling metadata → stage stats aggregation → serving Prometheus emission | | image/video/audio/action final type、registry alias、endpoint capability、formatter | [`output-runtime`](rules-output-lifecycle.md#diff-1s-diffusion-output-type-必须从模型声明闭合到-topology-与-formatter)：`DIFF-1s` | `model_metadata.py` → default/multi-stage config → video capability → `output_formatter.py` | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index cdb14a94f..5493d11b4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6b9ce05f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8bfad23f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 33b62b7e2..81bb0de69 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6b9ce05f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8bfad23f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 75d21b709..111983d1d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6b9ce05f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8bfad23f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index fd404638d..c420e7749 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6b9ce05f` 验证存在 +- 源码校验：以上路径均已在 `main @ 8bfad23f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 120175b48..6a08c3487 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6b9ce05f`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8bfad23f`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -130,8 +130,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 6d84409a \ - --to 6b9ce05f \ + --from 6b9ce05f \ + --to 8bfad23f \ --repo \ --mode report-only ``` From c233d65ab4639c35210b400fbb80f4f96eb5ac08 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:32:20 +0800 Subject: [PATCH 355/465] knowledge(vllm-omni): sync 034af1f6 (PR #6727) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 18 +++++++++++++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 15 +++++++++++---- 10 files changed, 37 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c5113e5e4..0c30a0a06 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6b9ce05febbb9ed1431c1570e967c6a2a85ff942 + previous_audited_sha: 8bfad23f98c118914bfedbaa46370d576c0add7d audited_ref: main - audited_sha: 8bfad23f98c118914bfedbaa46370d576c0add7d + audited_sha: 034af1f6570bc6156e998dae1a2db2de419e6466 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2b3c91f80..61eef603a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8bfad23f98c118914bfedbaa46370d576c0add7d`. The canonical machine baseline +034af1f6570bc6156e998dae1a2db2de419e6466`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 7f25d9963..058d74580 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730", "PR #6727"] --- # vLLM-Omni CI @@ -122,6 +122,22 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie configures no threshold, so it is benchmark-report/artifact collection rather than a performance regression gate. ^[PR #6349] +## XPU base-image resolution and support-table boundary + +- The Intel Buildkite XPU lane targets vLLM `v0.28.0`. Its Omni image layers on + `vllm/vllm-openai-xpu:${VLLM_VERSION}` and clears that image's serving entrypoint so the + result remains a shell/test container. The base carries the matching XPU runtime and vLLM; + Omni installs only its own layers. Keep the Dockerfile default and lane environment version + aligned. +- Before building, CI pulls the published upstream base freshly. If that pull fails it may use + an existing local copy; only if neither exists does it build the same vLLM tag from upstream + `docker/Dockerfile.xpu`, optionally under `VLLM_BASE`. Thus a fallback is provenance-aligned, + but it is not evidence that an arbitrary pre-existing fallback image matches the published + tag. The in-container scope remains the B60-marked core/advanced/omni pytest selections plus + `test_mxfp8_config.py`; it is not a whole-suite or model-matrix validation. The PR author + reported CI passing but supplied no command output or completed test plan, and review had no + surviving findings, so retain those evidence limits. ^[PR #6727] + ## AMD CI stabilization boundary - AMD merge lanes may combine READY and MERGE definitions only through the bootstrap's diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 15fcd535e..9007dc6ce 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8bfad23f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 034af1f6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index bf38e16ba..a8194ad2d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8bfad23f` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 034af1f6` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5493d11b4..597b76c16 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8bfad23f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 034af1f6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 81bb0de69..be659196e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8bfad23f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 034af1f6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 111983d1d..87316fed7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8bfad23f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 034af1f6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c420e7749..f20da72d7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8bfad23f` 验证存在 +- 源码校验：以上路径均已在 `main @ 034af1f6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6a08c3487..a363b4824 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -4,13 +4,13 @@ created: 2026-07-16 updated: 2026-09-05 type: guide tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/, "PR #5885"] +sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/, "PR #5885", "PR #6727"] --- # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8bfad23f`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 034af1f6`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -85,6 +85,13 @@ PR #6559 增加 `mimo_audio_5090d.yaml`，使 deploy inventory 83→84；这是 PR #4820 将 `WanDMDPipeline` 注册为 Wan2.2 的同实现入口，因此 diffusion pipeline 计数回升至 65；它不是新的模型族或新的 `OMNI_PIPELINES` key。^[PR #4820] +PR #6727 does not change any registry or deploy inventory, but the upstream supported-model +table adds Intel GPU checkmarks for Qwen-Image and Qwen-Image-2512, Krea 2, Wan2.2-VACE, +Boogu-Image, OmniVoice, VoxCPM2, Qwen3-TTS CustomVoice, Ming-omni-tts dense 0.5B, +Dynin-Omni, and Gepard-1.0. This is table-declared Intel GPU support; it is not a claim that +every model has a recipe, an online-serving validation, performance data, or validation on +hardware other than the PR's XPU lane. ^[PR #6727] + SANA-Video 2B 新增 native `SanaVideoPipeline`（T2V）和 `SanaImageToVideoPipeline`（I2V），二者共用 `sana_video` 模型族目录；它没有新增 `OMNI_PIPELINES` key 或 deploy YAML。模型专有的 checkpoint、VAE 和 adapter 边界见 @@ -130,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 6b9ce05f \ - --to 8bfad23f \ + --from 8bfad23f \ + --to 034af1f6 \ --repo \ --mode report-only ``` From 53ccb1edfd4271e7054c9401ffbc098f62ee7762 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:37:21 +0800 Subject: [PATCH 356/465] knowledge(vllm-omni): sync f584729f (PR #6622) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 4 ++-- .../vllm-omni/components/serving/rules.md | 18 ++++++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0c30a0a06..4cce168e5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8bfad23f98c118914bfedbaa46370d576c0add7d + previous_audited_sha: 034af1f6570bc6156e998dae1a2db2de419e6466 audited_ref: main - audited_sha: 034af1f6570bc6156e998dae1a2db2de419e6466 + audited_sha: f584729f2420e44a859580bf2b1f05cd6298998f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 61eef603a..c260fac90 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -034af1f6570bc6156e998dae1a2db2de419e6466`. The canonical machine baseline +f584729f2420e44a859580bf2b1f05cd6298998f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9007dc6ce..d3993c076 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 034af1f6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f584729f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a8194ad2d..74531fa34 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 034af1f6` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ f584729f` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 597b76c16..2464f4c87 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 034af1f6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f584729f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index be659196e..cb5d05841 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 034af1f6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f584729f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 87316fed7..0b3156ad6 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 034af1f6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f584729f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f20da72d7..abebe3ba3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -1,7 +1,7 @@ --- title: "Serving" created: 2026-07-10 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, components, serving] sources: [] @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 034af1f6` 验证存在 +- 源码校验：以上路径均已在 `main @ f584729f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index fb51be9d9..ea6e92f89 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -1,10 +1,10 @@ --- title: "Serving 规则" created: 2026-07-20 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_api_server_guards.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288", "PR #6622"] confidence: high --- @@ -239,6 +239,20 @@ confidence: high - 验收：同长度覆盖、并发改写、named+inline、speaker cache 和 prefix hit 均不复用旧内容； 非法路径不 stat，日志只含截断值且命中真实 logger hierarchy。 ^[PR #5670] +### SERV-3d — pure-diffusion speech 必须在 factory 固化 media policy + +- 触发：pure-diffusion speech 绕过普通 `OpenAIServing` 初始化，或修改 `ref_audio` 的 + `MediaConnector`、local-media allowlist 或 factory 参数。 +- 强制：app bootstrap 将 `allowed_local_media_path` 和 `allowed_media_domains` 原样传给 + speech factory；diffusion instance 在 factory 中创建并持有按该配置构造的 connector，且其 + local cache-key stat 使用同一 allowed path。普通 multi-stage instance 仅在首次需要时从 + `model_config` 构造 connector，不能要求 diffusion instance 拥有 `model_config`。 +- 禁止：diffusion 分支以无配置 connector 加载本地 `file:` URI；只把 allowlist 传到 + `MediaConnector` 而遗漏 cache-key metadata gate；每次解析重新构造 connector 并漂移策略。 +- 验收：pure-diffusion bootstrap 断言两个 media 参数到达 speech factory；在 allowlisted + 本地 ref-audio 上断言 connector 只按该配置创建一次，文件 size/mtime 改变后 cache miss 并 + refetch；普通 multi-stage 路径仍从 `model_config` 延迟构造。 ^[PR #6622] + ### SERV-11a — 用户输入日志必须默认降级并受限输出 - 触发：修改 TTS、audio generation 或 diffusion chat serving 请求日志，以及 `request_logger` 或 `max_log_len` 的行为。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a363b4824..82bf59e64 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 034af1f6`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f584729f`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 8bfad23f \ - --to 034af1f6 \ + --from 034af1f6 \ + --to f584729f \ --repo \ --mode report-only ``` From 3141bc9470d26e4f4b8e7e329599ff409e47d344 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:41:50 +0800 Subject: [PATCH 357/465] knowledge(vllm-omni): sync 55a8bd6b (PR #5831) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 15 ++++++++++++++- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/rules.md | 7 +++++-- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 31 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4cce168e5..f8a853c0d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 034af1f6570bc6156e998dae1a2db2de419e6466 + previous_audited_sha: f584729f2420e44a859580bf2b1f05cd6298998f audited_ref: main - audited_sha: f584729f2420e44a859580bf2b1f05cd6298998f + audited_sha: 55a8bd6b808e195ffd224d96698b18f61724b351 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c260fac90..137c91fa3 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f584729f2420e44a859580bf2b1f05cd6298998f`. The canonical machine baseline +55a8bd6b808e195ffd224d96698b18f61724b351`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 058d74580..e6ff625ca 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730", "PR #6727"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831"] --- # vLLM-Omni CI @@ -20,6 +20,19 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie implementation change. Reintroduce comparable coverage only with a passing CI-backed change. ^[PR #5464] ^[PR #6730] +## LTX-2 FP8 quality-gate recipe boundary + +- The H100 full-model `fp8_ltx2` gate compares Lightricks/LTX-2 online FP8 with BF16 using + the supported eager default recipe: 512×384, 73 frames, ten steps, seed 42, recipe-default + negative conditioning, and no request `sigmas` or `guidance_scale` override. Both arms keep + `enforce_eager=True`, so the LPIPS ≤ 0.20 threshold isolates quantization rather than compile + behavior. A CPU capturing test must assert that an absent negative prompt is omitted from the + request and that the sampling overrides remain `None`; the skipped gate was restored because + downstream RL training still relies on LTX-2. +- The PR reports one H100 run (LPIPS 0.0418, PSNR 33.0941 dB, MAE 0.017016) for that exact case. + It supports this gate's threshold, not a quantization implementation change, graph-mode parity, + other LTX variants, or a broader quality claim. ^[PR #5831] + ## MiniCPM native-duplex server-VAD coverage - Ready/merge 的 MiniCPM duplex source dependencies 纳入 server-VAD runner；其 L4 full-model diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d3993c076..67958b95b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f584729f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 55a8bd6b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 74531fa34..62c42d2e8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f584729f` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 55a8bd6b` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index d91aa18a6..22689eeba 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", "PR #5831", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] confidence: high --- @@ -176,11 +176,14 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl - 触发：新增 LPIPS/PSNR/相似度阈值，或用 CPU offload 等资源选项支撑量化质量测试。 - 强制：阈值证据与测试中的 step、seed、size、guidance、checkpoint 和量化组件完全一致； 仅为避免 OOM 的 offload 必须在 baseline/candidate 对称启用并说明它不属于待比较变量。 +- 当 case 声称使用 recipe default，必须让该 request key 真正缺席（不能以空值代替），并用 + capturing unit test 同时断言 prompt 和 sampling 参数都没有重新注入 override；若目标只是量化， + baseline/candidate 的 eager/compile mode 也必须保持相同。 - 禁止：用 50-step A/B 数字为 10-step 测试阈值背书；只写“需要 offload”而不说明 是资源前提还是功能行为。 - 验收：运行测试文件中的 exact case，并在规则/配置旁保留最短资源原因；对称 baseline 证明质量差异来自目标量化变量。低 steps smoke 只能证明 runtime compatibility，不得声称 - BF16 quality parity。 ^[PR #5136] ^[PR #6279] + BF16 quality parity。 ^[PR #5136] ^[PR #6279] ^[PR #5831] ### DIFF-4m — diffusion worker spawn 前必须限制继承的线程数 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 2464f4c87..eceb54950 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f584729f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 55a8bd6b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cb5d05841..125d7bfe3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f584729f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 55a8bd6b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0b3156ad6..081616e92 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f584729f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 55a8bd6b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index abebe3ba3..8d2e9c1d3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f584729f` 验证存在 +- 源码校验：以上路径均已在 `main @ 55a8bd6b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 82bf59e64..d4e382412 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f584729f`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 55a8bd6b`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 034af1f6 \ - --to f584729f \ + --from f584729f \ + --to 55a8bd6b \ --repo \ --mode report-only ``` From 243071fdf77b23a4846ef94626104e73203122bc Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:49:57 +0800 Subject: [PATCH 358/465] knowledge(vllm-omni): sync 9558fd2e (PR #6710) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/minimax-h3/_index.md | 2 ++ .../models/minimax-h3/rules-encoder-state.md | 18 ++++++++++++++++++ .../repos/vllm-omni/models/minimax-h3/rules.md | 1 + 12 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/rules-encoder-state.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f8a853c0d..ba0e90666 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f584729f2420e44a859580bf2b1f05cd6298998f + previous_audited_sha: 55a8bd6b808e195ffd224d96698b18f61724b351 audited_ref: main - audited_sha: 55a8bd6b808e195ffd224d96698b18f61724b351 + audited_sha: 9558fd2ea8d470203cc768b9ce4d9111d341c505 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 137c91fa3..29175b0f6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -55a8bd6b808e195ffd224d96698b18f61724b351`. The canonical machine baseline +9558fd2ea8d470203cc768b9ce4d9111d341c505`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 67958b95b..96c7c64f3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 55a8bd6b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9558fd2e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 62c42d2e8..e7cb1c8bf 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 55a8bd6b` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 9558fd2e` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index eceb54950..647bac20f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 55a8bd6b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9558fd2e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 125d7bfe3..f51b4fbef 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 55a8bd6b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9558fd2e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 081616e92..40f45aded 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 55a8bd6b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9558fd2e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8d2e9c1d3..4d37d5ca1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 55a8bd6b` 验证存在 +- 源码校验：以上路径均已在 `main @ 9558fd2e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d4e382412..838ac1d9f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 55a8bd6b`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9558fd2e`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from f584729f \ - --to 55a8bd6b \ + --from 55a8bd6b \ + --to 9558fd2e \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 7375dffd1..af4c7f0a6 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -142,3 +142,5 @@ text-encoder fused-source 完整性正文见 [loading rules](rules-loading.md) H3 VAE decoder 的 model-local eager dispatch、remote-code eligibility、exactness guards、selective FP16 materialization 与 compile/spatial-parallel fallback 见 [VAE eager-ops rules](rules-vae-ops.md)。 +Qwen3-VL text encoder 的 cuDNN SDPA process-global-state restore、encoder-rank/VAE boundary 和 +CPU-only validation limit 见 [encoder state rules](rules-encoder-state.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-encoder-state.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-encoder-state.md new file mode 100644 index 000000000..17bdd1a0f --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-encoder-state.md @@ -0,0 +1,18 @@ +--- +title: "MiniMax H3 encoder state rules" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #6710", vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, tests/diffusion/models/minimax_h3/test_minimax_h3_encoder_sdpa.py] +confidence: high +--- + +# MiniMax H3 encoder state rules + +## MMH3-1n — Qwen3-VL encoder 的 cuDNN SDPA override 必须恢复原状态 + +- 触发：修改 `MiniMaxH3Qwen3VLEncoder.encode_ids`、cuDNN SDPA backend flag、text-encoder TP，或 text-encoder→VAE 的执行边界。 +- 强制：本提交已捕获 `torch.backends.cuda.cudnn_sdp_enabled()` 的精确布尔值，并用 `finally` 在 `_encode()` 或 `hidden.cpu()` 失败及正常返回后恢复该值；文本编码期间仍启用 cuDNN SDPA。`text_encoder_tp_size` 小于 DiT world size 时，只有 encoder ranks 运行该调用；restore 防止它们将进程级 policy 留给随后所有 rank 参与的 VAE。该修复不为 video VAE 选择 Flash 或 cuDNN，VAE backend policy 仍须由各 rank 显式且对称地配置。 +- 禁止：只在初始 false 状态测试，恢复成硬编码 false，或把 CPU mock 的 flag assertion 描述成 cuDNN kernel、GPU output 或 multi-rank VAE parity 验证。当前实现仍在进入 `try` 前调用 `enable_cudnn_sdp(True)`；故 `_encode()`/`.cpu()` 失败时的泄漏已修复，但 enable 与 `try` 之间的中断或异常窗口仍是未关闭的 process-global-state limitation。将 enable 移入 `try` 是未来 hardening，不是本提交已提供的保证。 +- 验收：CPU unit tests 分别以初始 false/true 覆盖成功与 `_encode()` 抛错，断言 override 内为 true、退出后等于初始值；它们只证明 Python-level global-state save/restore。要声称 GPU kernel 或 SP/PP rank-policy parity，另在目标 CUDA 拓扑上观察每个相关 rank 的状态和实际 VAE execution。^[PR #6710] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 4c0e2be75..4b241f3cf 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -25,6 +25,7 @@ confidence: high | text encoder、missing q/k/v 或 gate/up、eager load bookkeeping | [MMH3-1f](rules-loading.md#mmh3-1f-text-encoder-eager-load-必须证明每个-source-shard-完整) | `encoder.py::_load_weights` → `pipeline_minimax_h3.py::load_weights` strict report | | NPU packed varlen、quadratic mask、LaserAttention、prefix K/V | `MMH3-1g` | H3 packed producer → backend capability/metadata → NPU FlashAttention fallback | | Qwen3-VL encoder、NPU causal GQA、AddRMSNorm residual | `MMH3-1k` | encoder SDPA helper/residual call → NPU model patch → `torch_npu` op | +| Qwen3-VL encoder、cuDNN SDPA、process-global backend state、encoder TP | [MMH3-1n](rules-encoder-state.md#mmh3-1n-qwen3-vl-encoder-的-cudnn-sdpa-override-必须恢复原状态) | `encode_ids` state capture/restore → encoder-rank call → VAE boundary | | VAE decoder、Triton exact ops、FP16 materialization、SM90/100/103、compile/SP fallback | [MMH3-4c](rules-vae-ops.md#mmh3-4c-h3-vae-eager-ops-必须以完整远程模型合同和-reference-fallback-安装) | `vae.py::MiniMaxH3VideoVAE.__init__` → `ops/vae::{dispatch,install}` → target tests | | FL2VA keyframe、Ref2VA mixed reference/时域限界、shape/output matrix | [MMH3-2a](rules-media.md#mmh3-2a-taskreferenceshape-与多输出必须作为一个输入矩阵维护) | `pipeline_minimax_h3.py` → `reference_video.py` | | media limit、typed/multipart reference、HTTP 400、temp source | [MMH3-2b](rules-media.md#mmh3-2b-media-ingress-在解码前限界request-错误保持-http-400) | `api_server.py` → `serving_video.py` → `reference_video.py` | From e9ec431773adff171f7626b19ab5e63fe6cc6822 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 01:57:57 +0800 Subject: [PATCH 359/465] knowledge(vllm-omni): sync d9406120 (PR #6327) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 5 +-- .../serving/rules-abort-lifecycle.md | 34 +++++++++++++++++++ .../serving/rules-engine-lifecycle.md | 4 +-- .../vllm-omni/components/serving/rules.md | 4 +-- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 12 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-abort-lifecycle.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ba0e90666..b81e778d8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 55a8bd6b808e195ffd224d96698b18f61724b351 + previous_audited_sha: 9558fd2ea8d470203cc768b9ce4d9111d341c505 audited_ref: main - audited_sha: 9558fd2ea8d470203cc768b9ce4d9111d341c505 + audited_sha: d94061201d99e64b94a00f734d32ef09d527b473 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 29175b0f6..b61ace609 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -9558fd2ea8d470203cc768b9ce4d9111d341c505`. The canonical machine baseline +d94061201d99e64b94a00f734d32ef09d527b473`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 96c7c64f3..be406b654 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9558fd2e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d9406120` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e7cb1c8bf..912d19207 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9558fd2e` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ d9406120` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 647bac20f..ab80b5823 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9558fd2e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d9406120` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f51b4fbef..62b8ced62 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9558fd2e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d9406120` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 40f45aded..b8144548a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9558fd2e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d9406120` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4d37d5ca1..5dc147fb2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9558fd2e` 验证存在 +- 源码校验：以上路径均已在 `main @ d9406120` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -32,7 +32,8 @@ sources: [] | upstream launcher shutdown、renderer warmup 与多 replica compile-cache 隔离 | [upstream 兼容规则](rules-upstream-compat.md) | | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | -| engine startup/shutdown、stage lifecycle、full-duplex、CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | +| engine startup/shutdown、stage lifecycle、acknowledged abort 的 shutdown race、full-duplex、CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | +| acknowledged abort 的 RPC ACK、queue/router shutdown race、cancellation cleanup 或 backpressure/timeout 保持 | [acknowledged abort 生命周期规则](rules-abort-lifecycle.md) | | assembled FastAPI route 覆盖、method/path ownership 或 request-body forwarding | [app assembly 规则](rules-app-assembly.md) | | stage config→EngineArgs projection、explicit `devices`、TP/local-DP/PP、replica layout 或 worker 创建前的 layout guard | [stage 启动与设备布局规则](rules-stage-startup.md) | | 公开请求字段的校验、来源冲突、alias/extras 归一与 consumer view | [请求输入合同](rules-request-input.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-abort-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-abort-lifecycle.md new file mode 100644 index 000000000..95f0de50c --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-abort-lifecycle.md @@ -0,0 +1,34 @@ +--- +title: "Serving acknowledged abort 生命周期规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #6327", vllm_omni/engine/async_engine_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/async_omni.py, tests/engine/test_async_omni_engine_abort_ack.py, tests/engine/test_async_omni_engine_input.py, tests/entrypoints/test_async_omni.py] +confidence: high +--- + +# Serving acknowledged abort 生命周期规则 + +本页收纳 `SERV-5t`；一般 engine lifecycle、stage shutdown 与 admission 合同见 +[engine 生命周期规则](rules-engine-lifecycle.md)。 + +### SERV-5t — acknowledged abort 在 shutdown 边界只忽略已识别的 transport closure + +- 触发：修改 `AsyncOmniEngine.abort()` / `abort_async()`、`CorrelatedRpcClient`、Janus request queue， + 或 `AsyncOmni.generate()` 的 cancellation cleanup。 +- 强制：空 request-ID batch 与已经观察到 `_shutdown_called` 的 late abort 是 no-op。live engine 的 + `abort_async()` 仍须以 correlated `rpc_id`、blocking request-queue submission 和 orchestrator ACK + 完成；不得削弱 queue backpressure 或 caller timeout。在 initial guard 后发生 race 时，只有再次确认 + shutdown 已开始，才可忽略 Janus 2.x `SyncQueueShutDown`、Janus 1.x 精确 closed-queue + `RuntimeError`，以及 ACK wait 的精确 RPC-result-router-closed error。成功 ACK 以前 frontend + request state 仍保留；shutdown 下 cancellation 仍沿既有 cleanup 收敛并清空 local request state。 +- 禁止：将所有 `RuntimeError`、live-engine queue/router closure、timeout、unexpected result type 或 + orchestrator abort failure 当作 shutdown 成功；以 plain `abort()` 取代 ACK path；在 ACK 前 pop + state，或为 cancellation 新增会 orphan live engine request 的无条件 cleanup。 +- 验收：覆盖 empty/late no-op、submit-time queue closure 和 ACK-wait router closure 的 + shutdown race；分别用 live-engine controls 断言同类 failure 继续 raise，并覆盖 Janus 2.x symbol + 缺失时的 legacy exact-message fallback。取消 `generate()` 后断言 original `CancelledError`、一次 + submission、无额外 abort enqueue 与 `request_states == {}`；另保留 ACK success、orchestrator + error、timeout 和 request-queue backpressure 的既有回归。PR 的 CPU tests 只证明这些 contract， + 不证明 hardware reliability 或任意 shutdown transport error 都安全忽略。^[PR #6327] diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 0811554bd..7634ee65f 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -1,7 +1,7 @@ --- title: "Serving engine 生命周期规则" created: 2026-09-03 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5221", "Issue #4855", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, tests/engine/test_orchestrator_event_driven.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py, "PR #6189", tests/entrypoints/test_async_omni_diffusion_config.py] @@ -10,7 +10,7 @@ confidence: high # Serving engine 生命周期规则 -本页收纳 `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5q`、`SERV-6a`–`SERV-6f`、 +本页收纳 `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5r`、`SERV-6a`–`SERV-6f`、 `SERV-8a` 与 `SERV-9a`。触发条件与其余审查组见 [Serving 共享规则](rules.md) 的 Direct 代码快速入口；请求输入侧的合同留在该页， 故障隔离见 [fault isolation 规则](rules-fault-isolation.md)。 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index ea6e92f89..075e91c71 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -23,7 +23,7 @@ confidence: high | batched chat、fan-out/fan-in、sub-request ID、choice collapse、whole-batch error | `batch-chat-contract`：`SERV-4i`–`4k` | `api_server.py::create_batch_chat_completion` → `batch_serving.py::OmniOpenAIServingChatBatch` → ordinary chat completion children | | `chat_template_kwargs`、raw HTTP/SDK `extra_body`、text/audio modalities、choices、空音频 | `chat-multimodal-contract`：`SERV-4c` + 命中模型规则 | upstream `ChatCompletionRequest` → `serving_chat.py::{OmniOpenAIServingChat._preprocess_chat,OmniOpenAIServingChat.chat_completion_full_generator,OmniOpenAIServingChat._create_text_choice,OmniOpenAIServingChat._create_audio_choice}` | | endpoint restriction、route/app-state guard、capability、公开 400 | `endpoint-capability`：`SERV-4c`, `SERV-4d`, `SERV-5d`, [SERV-5s](rules-app-assembly.md#serv-5s-覆盖-upstream-http-route-前必须先移除同-methodpath-的旧-route) | endpoint policy → `api_server.py::build_app` assembled app → public handler | -| pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup acknowledgment | `engine-lifecycle`：`SERV-5a`, `SERV-5b`, `SERV-5m` | `entrypoints/async_omni.py::{AsyncOmni.pause_generation,AsyncOmni.resume_generation,AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni._abort}` → `engine/{stage_pool,orchestrator,async_omni_engine}.py` → AR EngineCore helper / diffusion worker RPC | +| pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup acknowledgment 或 shutdown-time abort race | `engine-lifecycle`：`SERV-5a`, `SERV-5b`, `SERV-5m`；[SERV-5t](rules-abort-lifecycle.md#serv-5t-acknowledged-abort-在-shutdown-边界只忽略已识别的-transport-closure) | `entrypoints/async_omni.py::{AsyncOmni.pause_generation,AsyncOmni.resume_generation,AsyncOmni.sleep,AsyncOmni.wake_up,AsyncOmni._abort}` → `engine/{async_engine_utils,stage_pool,orchestrator,async_omni_engine}.py` → AR EngineCore helper / diffusion worker RPC | | stage/replica death、remote membership register/watch/detach、`EngineDeadError`、`StageUnavailableError`、health/readiness、eviction | `engine-lifecycle`：[SERV-5f](rules-fault-isolation.md#serv-5f-stage-death-按-replica-隔离request-fatal-不等于-process-fatal)、[SERV-5n](rules-engine-lifecycle.md#serv-5n-remote-replica-attachdetach-必须按本次-membership-generation-判定) | `engine/membership_controller.py` register/watcher generation → `engine/stage_pool.py` live set → `engine/orchestrator.py` poll/dispatch/cleanup → `entrypoints/{omni_base,async_omni}.py` error transport | | serving class/factory 重构、optional adapter、diffusion/no-TTS 实例、warmup | `engine-lifecycle`：`SERV-5c` | `entrypoints/openai/serving_speech.py` 的所有 factory/`__new__` 路径 → `warmup`、voice upload/list、speech request caller | | vLLM rebase、chat parser/renderer API moved or removed | `engine-lifecycle` | target-release `vllm.parser.*` / renderer API → `serving_chat.py` / `api_server.py`; remove a compatibility fallback only after the pinned base image is the target release ^[PR #6606] | @@ -46,7 +46,7 @@ confidence: high | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 见 engine lifecycle；`SERV-5s` 见 [app assembly](rules-app-assembly.md) | -| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup、streaming raw terminal、event-driven orchestration、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5q`（见 [engine 生命周期规则](rules-engine-lifecycle.md)），`SERV-5f` | +| `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup/shutdown race、streaming raw terminal、event-driven orchestration、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5r`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）、[SERV-5t](rules-abort-lifecycle.md#serv-5t-acknowledged-abort-在-shutdown-边界只忽略已识别的-transport-closure)，`SERV-5f` | | `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds、transform artifact ownership | `SERV-4a`–`4h`, `SERV-4l`–`4q`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 838ac1d9f..3a4b3d221 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9558fd2e`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d9406120`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 55a8bd6b \ - --to 9558fd2e \ + --from 9558fd2e \ + --to d9406120 \ --repo \ --mode report-only ``` From c4df2d5ac261aba2634b528647c19a2a1331b5b1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 02:07:54 +0800 Subject: [PATCH 360/465] knowledge(vllm-omni): sync 0dc8f459 (PR #6518) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-attention.md | 22 ++++++---- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 3 +- .../model-executor/rules-npu-profiler.md | 22 ++++++++++ .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../vllm-omni/models/minimax-h3/_index.md | 2 +- .../models/minimax-h3/rules-rainfusion.md | 40 +++++++++++++++++++ .../vllm-omni/models/minimax-h3/rules.md | 27 +------------ 14 files changed, 91 insertions(+), 47 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/model-executor/rules-npu-profiler.md create mode 100644 knowledge/repos/vllm-omni/models/minimax-h3/rules-rainfusion.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b81e778d8..7ff0e97d9 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9558fd2ea8d470203cc768b9ce4d9111d341c505 + previous_audited_sha: d94061201d99e64b94a00f734d32ef09d527b473 audited_ref: main - audited_sha: d94061201d99e64b94a00f734d32ef09d527b473 + audited_sha: 0dc8f459066713cee6632b0fd49750f6a7b8f4a4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b61ace609..1ae72950f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d94061201d99e64b94a00f734d32ef09d527b473`. The canonical machine baseline +0dc8f459066713cee6632b0fd49750f6a7b8f4a4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index be406b654..ed0c3606d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d9406120` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0dc8f459` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 912d19207..6414468a4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d9406120` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 0dc8f459` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 4b3744e72..9f2207e15 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -1,10 +1,10 @@ --- title: "Diffusion attention 规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] +sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6518", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] confidence: high --- @@ -81,9 +81,10 @@ confidence: high - 触发：修改 `RAINFUSION_ATTN`、`rf_v2`、`VideoTokenLayout`、block size、MindIE-SD 依赖或 packed video shape。 -- 强制：resolver 传入真实 `prefix_len`、`latent_shape=[t,h,w]` 和 - `used_len=prefix_len+t*h*w`；只裁掉 document-0 后的物理 padding，kernel 输出再补零回 - query shape。video rows 不再要求 128 整除；新 MindIE-SD 在空间重排后把不规则的 +- 强制：legacy resolver 传入真实 `prefix_len`、`latent_shape=[t,h,w]` 和 + `used_len=prefix_len+t*h*w`；Ref2VA multi-span resolver 改传 `used_len` 与按 physical start 排序的 + `video_spans=[{start, latent_shape}]`，将 interleaved text/image/audio 与 padding 保留 dense。只裁掉 + document-0 后的物理 padding，kernel 输出再补零回 query shape。video rows 不再要求 128 整除；新 MindIE-SD 在空间重排后把不规则的 真实 video suffix 提升到 always-kept segment，使余下 video 按 128-row block 稀疏。 不能用人工 padding 替代，因为 `rf_v2` 不消费 padding mask，pad key 会污染 softmax。 - 边界：`sparsity<=0`、未到 `start_step`、skip layer、无/未声明 BSND layout、无 @@ -91,12 +92,17 @@ confidence: high 32×128 rows 均保持 FlashAttention dense fallback；显式错 layout、causal 或 Ring>1 仍 fail closed。非 8×8 空间网格的 residual 被 always-kept，因而 realized sparsity 低于 nominal；潜空间 h/w 为 8 的倍数（输入高宽为 256 的倍数）仍是性能首选。 -- 禁止：只因 `mindiesd` 可 import 就断言 irregular-tail 合同可用；当前 availability - 只检查 module 存在，没有 version/feature gate，旧 `rf_v2` 与新 planner 组合仍有风险。 +- 禁止：只因 `mindiesd` 可 import 就断言 irregular-tail 或 multi-span 合同可用；availability 必须在 + 模型构造前检查 `sparse_attention` 明确接受 `video_spans`，否则以可操作的 upgrade/FLASH_ATTN error + fail closed，不能成功启动后在 T2VA/FL2VA/Ref2VA dispatch 才 TypeError，也不能静默尝试 legacy + signature。该 feature check 不是版本 pin，仍不能据此推断任意 MindIE-SD build 的 kernel quality。 - 验收：CPU plan 同时接受 128-aligned 与 irregular grid，并保持所有其他 fallback/ rejection；真实 Ascend + updated MindIE-SD 用 irregular grid 对 dense reference。目标 NPU test 仅以 `sparsity=0` 证明 protected-tail 几何下全 block 等价，不证明稀疏选择的质量、 - realized sparsity 或加速。^[PR #6000] + realized sparsity 或加速。CPU tests 还应覆盖 multi-span 的 used length、geometry/role/overlap/bounds、 + total threshold 与 clip-boundary dense-context fallback；它们只证明 plan payload。PR #6518 review 的 + MindIE-SD `2661f83` field-name/layout check 与 4×B300 FLASH_ATTN regression 都不执行 RainFusion； + 真实 Ref2VA same-seed sparse quality 仍未覆盖。^[PR #6000] ^[PR #6518] ## DIFF-1k — 非等长 SP attention 必须保留 K/V mask 并走 mask-safe backend diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ab80b5823..cec9bb665 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d9406120` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0dc8f459` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 62b8ced62..fbb3e3870 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,10 +13,11 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d9406120` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0dc8f459` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 +- NPU trace 的 profiler metric、handler 和证据边界见 [NPU profiler rules](rules-npu-profiler.md)。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-npu-profiler.md b/knowledge/repos/vllm-omni/components/model-executor/rules-npu-profiler.md new file mode 100644 index 000000000..e8ba922e3 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-npu-profiler.md @@ -0,0 +1,22 @@ +--- +title: "NPU profiler 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, model-executor] +sources: ["PR #6518", vllm_omni/platforms/npu/profiler.py] +confidence: high +--- + +# NPU profiler 规则 + +## EXEC-13g — NPU profiler metric 必须显式保留 trace 的利用率语义 + +- 触发：修改 `NPUTorchProfilerWrapper`、`torch_npu.profiler._ExperimentalConfig` 或 NPU trace 默认项。 +- 强制：默认 `aic_metrics` 使用 `AiCMetrics.PipeUtilization`，令导出的 Text/tensorboard trace 包含 + operator-pipeline utilization；保留 Level1、CPU+NPU activities、offline `analyse()` 提示和现有 trace + directory/handler 合同。 +- 禁止：把这个 telemetry default 说成 RainFusion/MiniMax-H3 kernel 的性能或质量证据，或由它推断任何 + Ascend SKU 的 utilization 已测得；它影响所有 NPU profiling run，不能静默改回无 AiCore metric。 +- 验收：mock `torch_npu` profiler config，精确断言 metric enum、活动与 handler 参数；真实 NPU trace + 另验证工具版本和可解析指标。PR #6518 没有提交该类 profiler artifact。^[PR #6518] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b8144548a..bdc5cac62 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d9406120` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0dc8f459` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5dc147fb2..9cf27fdbd 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d9406120` 验证存在 +- 源码校验：以上路径均已在 `main @ 0dc8f459` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3a4b3d221..35ad5082a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d9406120`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0dc8f459`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 9558fd2e \ - --to d9406120 \ + --from d9406120 \ + --to 0dc8f459 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index af4c7f0a6..97de75803 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -77,7 +77,7 @@ confidence: high guard 均回原 operation；完整合同见 MMH3-4c。^[PR #6607] - Ascend NPU 可选择 RainFusion 稀疏 video tail，并从 BF16 checkpoint 做 online INT8；两者只在 exact T2VA/Ulysses/no-AllGather DLO 配置有完成证据，几何 fallback、TP width 与组合边界见 - MMH3-1a/1d。 + MMH3-1a/[RainFusion rules](rules-rainfusion.md)。 - Ascend NPU `FLASH_ATTN` 的 H3 non-Ring packed path 可 opt in mask-free varlen，Laser env 改走 prefix K/V slice + 256 input scale；shared fallback/metadata 与 target TeaCache regression 见 MMH3-1g/DIFF-1g，有界 kernel/E2E/HBM 证据见 MMH3-3g。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-rainfusion.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-rainfusion.md new file mode 100644 index 000000000..6f85a48bc --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-rainfusion.md @@ -0,0 +1,40 @@ +--- +title: "MiniMax H3 RainFusion 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5706", "PR #6000", "PR #6518", vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py] +confidence: high +--- + +# MiniMax H3 RainFusion 规则 + +## MMH3-1d — RainFusion 只在已验证的 H3 video span 上稀疏 + +- 触发：修改 `RAINFUSION_ATTN`、`block_sparse`、packed video geometry、attention layout、denoise + step/layer skip 或 NPU backend resolution。 +- 强制：显式选择时所有平台在构造模型前检查 backend platform/dependency；RainFusion 仅支持 Ascend + NPU、要求 MindIE-SD，且不兼容 Ring（用 Ulysses）。H3 attention 显式声明 BSND。`VideoTokenLayout` + 有两种 typed、tensor-free producer contract：legacy `(prefix_len, latent_grid)` 必须证明单个 video + 是 packed document 0 的 tail；Ref2VA 则发布 `used_len` 和每个 physical + `VideoTokenSpan(start, latent_grid, role)`，其中 role 是 `reference|target`。H3 packer 只发布实际 + video grid，不把 image/audio/padding 放进 span；denoise setup 一次性把 producer 的 plain values + 转成 typed spans，禁止每层/每 step 从 device tensor 取标量。 +- 强制：只有 sparsity>0、step>=start_step、非 skip layer 且 video>=32×128 rows 时调用 `rf_v2`； + prefix/text/reference/audio 保持 dense。无 layout、无 `max_seqlen_q`、长度不闭合、短序列或未声明 + layout 仍走 NPU Flash dense fallback。multi-span 必须 `used_len==max_seqlen_q`、正的三维 grid、按 + start non-overlap/in-bounds、恰有一个 target，所有 video rows 合计达到 32×128；并且 non-contiguous + clips 之间的 dense context 足以隔离每个 preceding clip 的 128-token tail block。任一失败保持 dense， + 不能合并或猜测 spans。video rows 不被 128 整除时也传给 updated MindIE-SD，由它将 irregular real-video + suffix 纳入 always-kept 段；vLLM 不 padding。shared planner 和 dependency 边界见 DIFF-1h。 +- 禁止：把 nominal sparsity 当 realized sparsity 或质量保证；未声明 layout 时不能让 sparse path 假设 + BSND 而 dense fallback 解释成 BNSD。INT8、RainFusion、no-AllGather DLO 可组合，但 online quantization + 不得与 DLO+AllGather 组合；本 PR 没有证明 HSDP 或其他 quantizer 的组合语义。 +- 验收：CPU plan tests 覆盖 aligned/irregular 均进 sparse plan、tail closure、min length、layout 和 + skip/step；NPU 条件数值测试以 `sparsity=0` 直接调用 kernel 对 dense reference，mean relative error + 阈值为 `2e-3`；它不证明 `sparsity=0.8` 质量 parity。PR #6518 的 CPU tests 覆盖 Ref2VA multi-span + plan/packing 与旧 MindIE-SD startup rejection，但 review 没有同 seed 的 Ref2VA sparse(0.8) 对 dense + NPU quality comparison。PR body 所报 800I A2 单次 1.62×（1958.81→1208.90 s）和 A2/A3/A5 “verified” + 没有可审计的环境、raw artifact、repeats 或质量阈值，不能作为通用硬件/性能结论。MindIE-SD 也只 + feature-gate `video_spans` signature，未 pin version/commit。^[PR #5706] ^[PR #6000] ^[PR #6518] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 4b241f3cf..b0b70706b 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -20,7 +20,7 @@ confidence: high | grouped QKV、fused MLP、weight loader、TP | [MMH3-1a](rules-loading.md#mmh3-1a-component-namespace-与-checkpoint-transform-必须在-active-loader-前闭合) | `minimax_h3_transformer.py::MiniMaxH3DiTModel.load_weights` → active vLLM loader | | FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | | RMSNorm、RoPE、96/128 rotary dim、fused backend | `MMH3-1c` | `MiniMaxH3Attention` → shared `RMSNorm`/`RotaryEmbedding` → platform dispatch | -| RainFusion、block sparse、video layout、NPU INT8 | `MMH3-1d` | packed sequence → attention metadata/backend plan；quant config → prefixed linear → loader/post-load | +| RainFusion、block sparse、video layout、NPU INT8 | [MMH3-1d](rules-rainfusion.md#mmh3-1d-rainfusion-只在已验证的-h3-video-span-上稀疏) | packed sequence → attention metadata/backend plan；quant config → prefixed linear → loader/post-load | | TRTLLM、ragged packed metadata、SAGE、Skip-Softmax、Blackwell default | `MMH3-1e` | H3 metadata/roles → TRTLLM packed trim/quant gate → platform default | | text encoder、missing q/k/v 或 gate/up、eager load bookkeeping | [MMH3-1f](rules-loading.md#mmh3-1f-text-encoder-eager-load-必须证明每个-source-shard-完整) | `encoder.py::_load_weights` → `pipeline_minimax_h3.py::load_weights` strict report | | NPU packed varlen、quadratic mask、LaserAttention、prefix K/V | `MMH3-1g` | H3 packed producer → backend capability/metadata → NPU FlashAttention fallback | @@ -99,31 +99,6 @@ confidence: high 当前自动测试没有 MUSA dispatch/parity，CPU reference 只验证 96 维旋转和 32 维 passthrough。^[PR #5881] -## MMH3-1d — RainFusion 只在完整 H3 video tail 上稀疏 - -- 触发：修改 `RAINFUSION_ATTN`、`block_sparse`、packed video geometry、attention layout、 - denoise step/layer skip 或 NPU backend resolution。 -- 强制：显式选择时所有平台在构造模型前检查 backend platform/dependency；RainFusion 仅支持 - Ascend NPU、要求 MindIE-SD，且不兼容 Ring（用 Ulysses）。H3 attention 显式声明 BSND， - `VideoTokenLayout(prefix_len, latent_grid)` 必须证明 video 是 packed document 0 的 tail。 -- 强制：只有 sparsity>0、step>=start_step、非 skip layer 且 video>=32×128 rows 时调用 - `rf_v2`；prefix/text/reference/audio 保持 dense。无 layout、无 `max_seqlen_q`、长度不闭合、 - 短序列或未声明 layout 仍走 NPU Flash dense fallback。video rows 不被 128 整除时 - 也传给 updated MindIE-SD，由它将 irregular real-video suffix 纳入 always-kept 段； - vLLM 不 padding。shared planner 和 dependency 边界见 DIFF-1h。 -- 禁止：把 nominal sparsity 当 realized sparsity或质量保证；未声明 layout 时不能让 sparse path - 假设 BSND 而 dense fallback 解释成 BNSD。INT8、RainFusion、no-AllGather DLO 可组合，但 online - quantization 不得与 DLO+AllGather 组合；本 PR 没有证明 HSDP 或其他 quantizer 的组合语义。 -- 验收：CPU plan tests 覆盖 aligned/irregular 均进 sparse plan、tail closure、min length、layout - 和 skip/step； - NPU 条件数值测试以 `sparsity=0` 直接调用 kernel 对 dense reference，mean relative error 阈值 - 为 `2e-3`；它不证明 `sparsity=0.8` 质量 parity。PR 的 Atlas 800I A3 8×NPU、 - CANN 9.0.1、T2VA 209-frame 三种生成仅证明这些 exact 配置能完成；视频样例不是质量阈值， - 没有 latency/repeats，不能宣称稳定加速或 FL2VA/Ref2VA 支持。1344×768 - H3 grid `(62,24,42)` 的 62496 video rows 只在 CPU plan test 证明会进 sparse plan； - updated-MindIE NPU 对照用更小的 `(4,16,10)` grid 且 `sparsity=0`，仍不证明 0.8 - 的质量或性能。^[PR #5706] ^[PR #6000] - ## MMH3-1e — H3 TRTLLM 必须从 packed 结构裁掉 padding 并隔离短序列 role - 触发：修改 H3 的 TRTLLM default、packed `cu_seqlens`/mask、SAGE quant、Skip-Softmax、 From 8e7f850291dceb0b63af8abc6663428eb2051917 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 02:16:16 +0800 Subject: [PATCH 361/465] knowledge(vllm-omni): sync 46cfa843 (PR #6499) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-output-lifecycle.md | 8 ++++---- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 11 ++++++----- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 22 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 7ff0e97d9..92b08e909 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d94061201d99e64b94a00f734d32ef09d527b473 + previous_audited_sha: 0dc8f459066713cee6632b0fd49750f6a7b8f4a4 audited_ref: main - audited_sha: 0dc8f459066713cee6632b0fd49750f6a7b8f4a4 + audited_sha: 46cfa8435b76b57b3e354654212ae16d8a513f1a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1ae72950f..4ef8c9e05 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0dc8f459066713cee6632b0fd49750f6a7b8f4a4`. The canonical machine baseline +46cfa8435b76b57b3e354654212ae16d8a513f1a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ed0c3606d..0c6722c21 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0dc8f459` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 46cfa843` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6414468a4..e48de81ab 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0dc8f459` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 46cfa843` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index fca5f9f4e..bf99b8a1c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -1,10 +1,10 @@ --- title: "Diffusion output 与 multiprocess runtime 规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308"] +sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499"] confidence: high --- @@ -112,9 +112,9 @@ confidence: high ## DIFF-9a — 共享 PyAV 预构造帧 mux 必须闭合资源与音频时间零点 - 触发：修改共享 diffusion 的预构造 PyAV 视频帧 mux、可选音频 mux、container cleanup 或编码器异常传播。 -- 强制：`mux_av_video_audio_bytes()` 必须在 context manager 内打开并配置 MP4 video/audio streams，消费 `Iterable[av.VideoFrame]` 后 flush video encoder；音频按 `fltp` 和 mono/stereo layout 构造，并以 `pts=0`、`time_base=1/sample_rate` 标记输入时间零点。成功和 generator、encoder、mux 异常都必须退出 context 关闭 container，同时原样传播原始错误。 +- 强制：`mux_av_video_audio_bytes()` 必须在 context manager 内打开并配置 MP4 video/audio streams，消费 `Iterable[av.VideoFrame]` 后 flush video encoder；音频按 `fltp` 和 mono/stereo layout 构造，并以 `pts=0`、`time_base=1/sample_rate` 标记输入时间零点。`audio_sample_rate` 可为 `None`：仅在 audio waveform 存在时内部解析为 44.1 kHz，video-only 不创建 audio stream，也不需要采样率。成功和 generator、encoder、mux 异常都必须退出 context 关闭 container，同时原样传播原始错误。 - 禁止：在 generator 消费前或 context 外打开/使用 container；只在成功路径手工 close；吞掉帧生成或编码异常后静默改走其他 muxer；把 mock container cleanup 或单一 CPU 运行误认为完整跨硬件 A/V parity。 -- 验收：测试覆盖预构造 `gbrp` 帧的 H.264 mux、无音频/mono/stereo 音频、AAC 首个有效 packet 的负 priming PTS、video flush 和 container close；让 frame generator 在中途抛错并断言 container 已关闭且异常 identity 保持，再用固定媒体输入执行 ffprobe 与完整 FFmpeg 解码。^[PR #6288] +- 验收：测试覆盖预构造 `gbrp` 帧的 H.264 mux、无音频/mono/stereo 音频、`None` rate 的 video-only 与 audio 44.1 kHz default、AAC 首个有效 packet 的负 priming PTS、video flush 和 container close；让 frame generator 在中途抛错并断言 container 已关闭且异常 identity 保持，再用固定媒体输入执行 ffprobe 与完整 FFmpeg 解码。^[PR #6288] ^[PR #6499] ## DIFF-11a — 单 GPU diffusion executor 必须保持进程内故障与生命周期语义 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index cec9bb665..ea3b16e35 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0dc8f459` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 46cfa843` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index fbb3e3870..48c206046 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0dc8f459` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 46cfa843` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bdc5cac62..e47633d17 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0dc8f459` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 46cfa843` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9cf27fdbd..cd3b70034 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0dc8f459` 验证存在 +- 源码校验：以上路径均已在 `main @ 46cfa843` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 075e91c71..c61263125 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_api_server_guards.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288", "PR #6622"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_api_server_guards.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288", "PR #6622", "PR #6499"] confidence: high --- @@ -33,7 +33,7 @@ confidence: high | caller sampling params、pipeline `sampling_constraints`、stage config runtime extraction | `request-contract`：`SERV-4o` | `OmniBase.resolve_sampling_params_list` → `_get_sampling_constraints_list` → `_apply_sampling_constraints` | | OpenAI integer bounds、msgpack overflow、request-local cleanup | `request-contract`：`SERV-4p` | protocol request models → orchestrator stage dispatch → `_fail_request_client_error` | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | -| video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | +| video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e`, `1k` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | | Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle、image/diffusion metric emission | `metrics-lifecycle`：`SERV-2a`, `SERV-2b`, `SERV-2e` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | @@ -154,9 +154,10 @@ confidence: high ## SERV-1k — 非流式 MP4 编码必须按运行时能力自动选择并保留回退语义 - 触发：修改非流式 MP4 视频响应的 normalize、frame layout/dtype 判断、mux、raw/base64 输出或 fallback 行为。 -- 强制：公共 dispatcher 先完成一次帧准备和形状校验；仅当所有帧具有相同的正 HWC 形状、3/4 通道、支持的 common dtype（`uint8`、bool 或浮点）且 RGB channel view 均为 C-contiguous 时选择 `direct_planar`，逐通道量化到 PyAV 的 GBR planar 帧并复用单个 scratch buffer；其他输入必须在打开 PyAV container 前选择 `legacy_fallback`，复用已准备帧并记录稳定 fallback reason 与有效媒体参数。raw MP4 和 base64 共用该 dispatcher，不提供 CLI、模型声明或请求级编码策略；流式 fMP4 保持原路径。 -- 禁止：为能力探测再次 normalize/copy 视频；在 direct mux 开始后因异常重试 legacy；把 interleaved、非法形状或不支持 dtype 静默送入 planar path；把 Kunpeng/Atlas 测量外推为通用硬件性能或 DiT 加速。 -- 验收：覆盖 channel-first-backed direct path、interleaved/unsupported shape/dtype fallback、RGB contiguity、GBR plane 与 padded stride、direct failure 不重试、音频存在且采样率缺省时仅一次使用 24 kHz 日志、显式采样率和 video-only 日志；固定输入下有无音频的 direct/legacy MP4 必须 byte-identical 并通过 ffprobe/完整 FFmpeg decode，raw/base64 不携带 policy config，streaming 行为不变。^[PR #6288] +- 强制：公共 dispatcher 先完成一次帧准备和形状校验；仅当所有帧具有相同的正 HWC 形状、3/4 通道、支持的 common dtype（`uint8`、bool 或浮点）且 RGB channel view 均为 C-contiguous 时选择 `direct_planar`，逐通道量化到 PyAV 的 GBR planar 帧；其他输入必须在打开 PyAV container 前选择 `legacy_fallback`，复用已准备帧并记录稳定 fallback reason 与有效媒体参数。raw MP4 和 base64 共用该 dispatcher，不提供 CLI、模型声明或请求级编码策略；流式 fMP4 保持原路径。在线 video handler 拥有一个可复用的 8-worker `_PlanarFrameConverter`，两条非流式响应路径都传入它；utility caller 未传 converter 时以临时 serial converter 保持相同 builder 语义。 +- 强制：converter 的 scratch 是每个 worker thread 的实例本地存储；`iter_frames()` 按 FIFO 产生帧，并且一次调用最多保留 `2 * min(frame_count, max_workers)` 个 pending futures。无论 mux 成功还是抛错，调用方都必须关闭 iterator 以 cancel remaining futures；server `finally` 关闭 video handler，handler shutdown 必须幂等并以 `wait=True, cancel_futures=True` 关闭 executor。日志报告 effective frame-conversion workers（legacy 为 0）。 +- 禁止：为能力探测再次 normalize/copy 视频；在 direct mux 开始后因异常重试 legacy；把 interleaved、非法形状或不支持 dtype 静默送入 planar path；不得把 async route 仍同步执行 mux/base64 或单请求 pending 上限说成全请求 nonblocking 或所有并发请求的全局 pending 上限。共享 pool 只在一个 API process 内限制 conversion threads；多 process 仍各自拥有 pool，且性能证据仍限单请求。 +- 验收：覆盖 channel-first-backed direct path、interleaved/unsupported shape/dtype fallback、RGB contiguity、GBR plane 与 padded stride、serial/parallel exact output、FIFO 与单调用 future bound、iterator close 后 cancel、reused executor 的幂等 shutdown、direct failure 不重试；音频存在时 serving resolver 才以 24 kHz 为缺省，显式采样率和 video-only 日志保持正确。固定输入下有无音频的 direct/legacy MP4 必须 byte-identical 并通过 ffprobe/完整 FFmpeg decode，raw/base64 传同一 persistent converter，streaming 行为不变。^[PR #6288] ^[PR #6499] ## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 35ad5082a..8b033c102 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0dc8f459`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 46cfa843`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from d9406120 \ - --to 0dc8f459 \ + --from 0dc8f459 \ + --to 46cfa843 \ --repo \ --mode report-only ``` From 1ab62836acd08f830310ec821f332cc72f6ddcc2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 02:21:31 +0800 Subject: [PATCH 362/465] knowledge(vllm-omni): sync bbb3ae01 (PR #6743) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 20 +++++++++++++++++-- .../components/configuration/_index.md | 4 ++-- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 4 ++-- .../components/model-executor/_index.md | 4 ++-- .../vllm-omni/components/scheduler/_index.md | 4 ++-- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 35 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 92b08e909..95acf0e89 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0dc8f459066713cee6632b0fd49750f6a7b8f4a4 + previous_audited_sha: 46cfa8435b76b57b3e354654212ae16d8a513f1a audited_ref: main - audited_sha: 46cfa8435b76b57b3e354654212ae16d8a513f1a + audited_sha: bbb3ae01a10e18d749942067c4eaaff1134bf21a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4ef8c9e05..b0fc10060 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -46cfa8435b76b57b3e354654212ae16d8a513f1a`. The canonical machine baseline +bbb3ae01a10e18d749942067c4eaaff1134bf21a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index e6ff625ca..3da35116d 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743"] --- # vLLM-Omni CI diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 2471cc8f0..713602bbf 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni CI 规则" created: 2026-08-23 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704"] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743"] confidence: high --- @@ -258,6 +258,22 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload supported-format 提示。纯文本五并发的 reduced-token 配置独立覆盖 batching，不能因删除相邻 audio-only case 而丢失。^[PR #6570] +- Qwen3-Omni audio-output DFX baselines must select `mean_audio_rtf`, not + `mean_e2el_ms`, when the talker may free-run to its stop token: generated-audio + length changes e2el without establishing a serving-speed regression. At + `bbb3ae01`, this applies to the three async-chunk `random-mm` audio cases and + all four no-async-chunk audio cases (including the five-point `data=random` + sweep); their surviving H100 `mean_audio_rtf` limits are respectively + `[0.1098]`, `[0.1335]`, `[0.1798]`, `[0.169, 0.2041, 0.2382, 0.3177, 0.4595]`, + `[0.138]`, `[0.1609]`, and `[0.2048]`. Async-chunk cases with + `modalities: ["text"]` retain `mean_e2el_ms`, because they do not produce talker + audio. Do not confuse baseline metric selection for downstream comparison with + collection: `percentile-metrics` still includes `e2el` in every affected audio + case, so e2el remains reported but is no longer selected by the baseline. + This PR changes only perf configuration (25 deletions), not runtime behavior; + its local plan could validate JSON shape only because baseline comparison occurs + downstream after artifact upload. ^[PR #6743] + ## OMNI-CI-3b — patched upstream benchmark 必须保持参数与 output subtype 兼容 - 触发：upstream `benchmarks/serve.py` 增参，或 Omni 聚合字段只存在于扩展 output subtype。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0c6722c21..275ed3647 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni Configuration" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, components, config] sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 46cfa843` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bbb3ae01` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e48de81ab..03d19dfd8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 46cfa843` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ bbb3ae01` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ea3b16e35..af358e402 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -1,7 +1,7 @@ --- title: "Distributed（跨 stage 通信与数据搬运）" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, components, distributed] sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coordinator/, docs/design/feature/disaggregated_inference.md] @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 46cfa843` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bbb3ae01` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 48c206046..ec41c4ce1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -1,7 +1,7 @@ --- title: "Model Executor" created: 2026-07-10 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, components, model-executor] sources: [] @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 46cfa843` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bbb3ae01` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e47633d17..cd5c15f1a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -1,7 +1,7 @@ --- title: "Scheduler（AR/生成请求调度）" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, components, scheduler] sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/output.py, vllm_omni/core/prefix_cache.py, docs/design/module/ar_runtime.md, docs/design/module/archive/ar_module.md] @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 46cfa843` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bbb3ae01` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cd3b70034..544cbc040 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 46cfa843` 验证存在 +- 源码校验：以上路径均已在 `main @ bbb3ae01` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8b033c102..25469e528 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 46cfa843`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bbb3ae01`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 0dc8f459 \ - --to 46cfa843 \ + --from 46cfa843 \ + --to bbb3ae01 \ --repo \ --mode report-only ``` From eada04f57e8ba52ecac63c0e173be7a38decea19 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 02:30:13 +0800 Subject: [PATCH 363/465] knowledge(vllm-omni): sync d934a5f7 (PR #6549) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 3 +- .../components/serving/rules-metrics.md | 40 +++++++++++++++++++ .../vllm-omni/components/serving/rules.md | 29 +------------- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 11 files changed, 55 insertions(+), 39 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-metrics.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 95acf0e89..aa1b27f2b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 46cfa8435b76b57b3e354654212ae16d8a513f1a + previous_audited_sha: bbb3ae01a10e18d749942067c4eaaff1134bf21a audited_ref: main - audited_sha: bbb3ae01a10e18d749942067c4eaaff1134bf21a + audited_sha: d934a5f7b8829b970b7e642398dcd9e14038e4d1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b0fc10060..7d4d51cab 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bbb3ae01a10e18d749942067c4eaaff1134bf21a`. The canonical machine baseline +d934a5f7b8829b970b7e642398dcd9e14038e4d1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 275ed3647..7d75a7f44 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bbb3ae01` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d934a5f7` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 03d19dfd8..eeeebba00 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bbb3ae01` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ d934a5f7` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index af358e402..70cc3f288 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bbb3ae01` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d934a5f7` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ec41c4ce1..f2bf2f854 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bbb3ae01` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d934a5f7` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cd5c15f1a..c7e5865e9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bbb3ae01` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d934a5f7` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 544cbc040..ebace86a5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bbb3ae01` 验证存在 +- 源码校验：以上路径均已在 `main @ d934a5f7` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -29,6 +29,7 @@ sources: [] |---|---| | 理解入口到 engine 的边界 | [architecture](architecture.md) | | 根据 PR 描述直达 request contract、stage sampling constraints、streaming format、artifact readiness、factory lifecycle 或 metrics 的规则组与第一批源码 | [rules 与代码地图](rules.md) | +| Prometheus family、pipeline waiting/running gauge、stage/replica snapshot 或 collector lifecycle | [metrics 生命周期规则](rules-metrics.md) | | upstream launcher shutdown、renderer warmup 与多 replica compile-cache 隔离 | [upstream 兼容规则](rules-upstream-compat.md) | | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-metrics.md b/knowledge/repos/vllm-omni/components/serving/rules-metrics.md new file mode 100644 index 000000000..2bb56fe14 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-metrics.md @@ -0,0 +1,40 @@ +--- +title: "Serving metrics 生命周期规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #3576", "PR #4755", "PR #6549", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, tests/metrics/test_prometheus.py] +confidence: high +--- + +# Serving metrics 生命周期规则 + +## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 + +- 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 +- 强制：节流状态按实际 producer owner 隔离；request 状态 cleanup 后再计算 waiting 等 gauge。 +- 禁止：用一个全局时间戳让先上报的 replica 抑制其他 replica；在 pop/cleanup 前发布最终 gauge。 +- 验收：同一窗口内两个 replica 都能上报；单请求完成并清理后 waiting=0。^[PR #3576] + +## SERV-2b — collector 重建只保护本项目 family + +- 触发：同进程重建 engine、注册 Prometheus collectors 或调整 unregister 行为。 +- 强制：只保护需要跨实例保留的 `vllm:omni_*` family，并保留 upstream collector 的正常 unregister/cleanup。 +- 禁止：把 upstream unregister 整体置空，导致重复 timeseries 注册。 +- 验收：同进程连续创建/销毁两次 engine，无 duplicate-timeseries 错误且 Omni family 仍可采集。^[PR #3576] + +## SERV-2c — stage metrics 只能在 result message 首次消费时累计 + +- 触发：修改 `OmniBase` 的 result message 去重、`accumulate_diffusion_metrics`、`on_stage_metrics` 或重试/流式结果处理。 +- 强制：以每个请求的 `msg_id = id(result)` consumed 集合作为唯一门禁；仅在消息首次未消费时累计 diffusion metrics、处理 stage metrics，完成两项操作后再记录该消息已消费。 +- 禁止：在轮询重试、重复回调或消息标记后再次累计同一 result；把同一消息误当作新的 denoising step，或绕过现有 consumed 生命周期清理。 +- 验收：同一 result 重复处理只产生一次累计，两个 distinct result 各自产生一次；覆盖异常、重试与请求清理，确认 consumed 状态不会导致重复累计或遗留。^[PR #4755] + +## SERV-2f — pipeline gauge 必须把 engine 内 scheduler queue 从 running 移到 waiting + +- 触发：修改 `vllm_omni:num_requests_running` / `vllm_omni:num_requests_waiting`、orchestrator 的 stage/replica scheduler snapshot，或 `OmniBase` 的 request arrival、result/finalization、cleanup 发布点。 +- 强制：`AsyncOmniEngine` 与 `Orchestrator` 共享 `_engines_waiting_counter`；每次更新或移除非负的 `(stage_id, replica_id)` waiting snapshot 后，orchestrator 将其和同步进该 counter。前端只能经 `_publish_request_gauges(total)` 发布：`dispatched` 取 `_running_counter`（缺失时取 `total`），`engine_waiting` clamp 到 `[0, dispatched]`，`running = dispatched - engine_waiting`，`waiting = max(0, total - dispatched) + engine_waiting`。请求到达、stage result（包括 finalization 的 `total - 1` race 语义）和 cleanup 后都必须刷新，因此 cleanup 后的总数会覆盖尚未 pop 时的临时发布。 +- 禁止：把 orchestrator 已 dispatch、但仍在 stage scheduler queue 的请求报作 running；分散复制公式；让负值、过期/竞态 snapshot 或 engine waiting 大于 dispatched 产生负 gauge；在 final request 仍在 `request_states` 时丢掉既有 `total - 1` 防止 waiting 卡为 1 的语义。 +- 限制：该实现求的是各 `(stage, replica)` 最新 waiting snapshot 的**和**，并不按 request ID 去重；代码和 PR #6549 的 3-dispatched/2-queued 测试只证明单一 queue 的拆分，未证明同一 pipeline request 不会同时落在多个 snapshot。因此该值是受 clamp 保护的 scheduler-queue occupancy，不应声称是精确的全局唯一请求数。PR #6549 唯一 review thread 仅确认 `engines_` 名称用于区分 frontend 与 engine waiting，未处理该 cardinality 问题。 +- 验收：用真实 Prometheus scrape 覆盖 3 个已 dispatch、其中 2 个 engine scheduler waiting，精确断言 running=1、waiting=2；同时覆盖 arrival、普通/最终 stage result、cleanup、replica removal 和 snapshot 的负值/超 dispatched 值，确认既有 finalization race 语义不变。^[PR #6549] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index c61263125..b15789bbc 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -35,14 +35,14 @@ confidence: high | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | | video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e`, `1k` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | -| Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle、image/diffusion metric emission | `metrics-lifecycle`：`SERV-2a`, `SERV-2b`, `SERV-2e` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | +| Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle、pipeline request gauge 或 image/diffusion metric emission | [metrics-lifecycle rules](rules-metrics.md)：`SERV-2a`, `SERV-2b`, `SERV-2e`, `SERV-2f` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message,_publish_request_gauges}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| | `core` | 每次 serving 审查 | `SERV-4c`，见 [请求输入合同](rules-request-input.md) | | `streaming-format` | SSE、audio streaming、format/default/capability | `SERV-1a`, `SERV-1b` | | `media-ingress` | video reference、decoder registry/backend、mixed capability、bounded upload/conversion | `SERV-1c`–`1e` | -| `metrics-lifecycle` | metrics、gauge、replica、collector、image/diffusion measurement boundary | `SERV-2a`, `SERV-2b`, `SERV-2e` | +| `metrics-lifecycle` | metrics、gauge、replica、collector、pipeline request gauge 或 image/diffusion measurement boundary | [SERV-2a–2f](rules-metrics.md) | | `artifact-readiness` | artifact/content cache、capability、ready/mark/discard | `SERV-3a`, `SERV-3b`, `SERV-3c` | | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 见 engine lifecycle；`SERV-5s` 见 [app assembly](rules-app-assembly.md) | @@ -159,31 +159,6 @@ confidence: high - 禁止：为能力探测再次 normalize/copy 视频；在 direct mux 开始后因异常重试 legacy；把 interleaved、非法形状或不支持 dtype 静默送入 planar path；不得把 async route 仍同步执行 mux/base64 或单请求 pending 上限说成全请求 nonblocking 或所有并发请求的全局 pending 上限。共享 pool 只在一个 API process 内限制 conversion threads；多 process 仍各自拥有 pool，且性能证据仍限单请求。 - 验收：覆盖 channel-first-backed direct path、interleaved/unsupported shape/dtype fallback、RGB contiguity、GBR plane 与 padded stride、serial/parallel exact output、FIFO 与单调用 future bound、iterator close 后 cancel、reused executor 的幂等 shutdown、direct failure 不重试；音频存在时 serving resolver 才以 24 kHz 为缺省，显式采样率和 video-only 日志保持正确。固定输入下有无音频的 direct/legacy MP4 必须 byte-identical 并通过 ffprobe/完整 FFmpeg decode，raw/base64 传同一 persistent converter，streaming 行为不变。^[PR #6288] ^[PR #6499] -## SERV-2a — 指标节流和 gauge 按 scheduler/stage/replica owner 隔离 - -- 触发：orchestrator 聚合多 stage/replica stats，或新增全局 throttle/gauge。 -- 强制：节流状态按实际 producer owner 隔离；request 状态 cleanup 后再计算 waiting 等 - gauge。 -- 禁止：用一个全局时间戳让先上报的 replica 抑制其他 replica；在 pop/cleanup 前发布 - 最终 gauge。 -- 验收：同一窗口内两个 replica 都能上报；单请求完成并清理后 waiting=0。 ^[PR #3576] - -## SERV-2b — collector 重建只保护本项目 family - -- 触发：同进程重建 engine、注册 Prometheus collectors 或调整 unregister 行为。 -- 强制：只保护需要跨实例保留的 `vllm:omni_*` family，并保留 upstream collector 的 - 正常 unregister/cleanup。 -- 禁止：把 upstream unregister 整体置空，导致重复 timeseries 注册。 -- 验收：同进程连续创建/销毁两次 engine，无 duplicate-timeseries 错误且 Omni family - 仍可采集。 ^[PR #3576] - -## SERV-2c — stage metrics 只能在 result message 首次消费时累计 - -- 触发：修改 `OmniBase` 的 result message 去重、`accumulate_diffusion_metrics`、`on_stage_metrics` 或重试/流式结果处理。 -- 强制：以每个请求的 `msg_id = id(result)` consumed 集合作为唯一门禁；仅在消息首次未消费时累计 diffusion metrics、处理 stage metrics，完成两项操作后再记录该消息已消费。 -- 禁止：在轮询重试、重复回调或消息标记后再次累计同一 result；把同一消息误当作新的 denoising step，或绕过现有 consumed 生命周期清理。 -- 验收：同一 result 重复处理只产生一次累计，两个 distinct result 各自产生一次；覆盖异常、重试与请求清理，确认 consumed 状态不会导致重复累计或遗留。^[PR #4755] - ## SERV-2e — 指标发射必须保持测量边界、缺失语义与请求唯一性 - 触发：新增 image/diffusion Prometheus family，或修改 `OmniBase` terminal result、 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 25469e528..fdde0a1fb 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bbb3ae01`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d934a5f7`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 46cfa843 \ - --to bbb3ae01 \ + --from bbb3ae01 \ + --to d934a5f7 \ --repo \ --mode report-only ``` From 8982aeee44cce590ce6daf6ed30223224bf937e4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 02:37:35 +0800 Subject: [PATCH 364/465] knowledge(vllm-omni): sync fbc93a0e (PR #6742) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 14 +++++++++++++- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minimax-h3/_index.md | 7 ++++++- 11 files changed, 31 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index aa1b27f2b..2cfd5af89 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bbb3ae01a10e18d749942067c4eaaff1134bf21a + previous_audited_sha: d934a5f7b8829b970b7e642398dcd9e14038e4d1 audited_ref: main - audited_sha: d934a5f7b8829b970b7e642398dcd9e14038e4d1 + audited_sha: fbc93a0e08e44bd92d68c8c12617642f47448990 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7d4d51cab..983b079de 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d934a5f7b8829b970b7e642398dcd9e14038e4d1`. The canonical machine baseline +fbc93a0e08e44bd92d68c8c12617642f47448990`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 3da35116d..07b627fb3 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742"] --- # vLLM-Omni CI @@ -33,6 +33,18 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie It supports this gate's threshold, not a quantization implementation change, graph-mode parity, other LTX variants, or a broader quality claim. ^[PR #5831] +## MiniMax-H3 two-GPU FP8 quality-test initialization boundary + +- The full-model CUDA quality test shares one `common_kwargs` mapping between its fused BF16 + baseline and transformer-FP8 candidate: `enforce_eager=True`, `tensor_parallel_size=2`, + `text_encoder_tp_size=2`, and VAE tiling. On the exact two-card H100-80GB scope, encoder TP + is required for the fused BF16 baseline to initialize; without it, rank 0 retains the full + colocated text encoder and can OOM before the FP8 comparison begins. +- This restores a CI test topology/resource setting only. The author reported one local two-B300 + run at the stated H100-marked selection, but no completed H100-80GB Buildkite result is + recorded here; therefore it does not establish an H100 pass, a result on other hardware, a + broader memory bound, or a runtime/quantization implementation change. ^[PR #6742 / issue #6735] + ## MiniCPM native-duplex server-VAD coverage - Ready/merge 的 MiniCPM duplex source dependencies 纳入 server-VAD runner；其 L4 full-model diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7d75a7f44..069e5ac3e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d934a5f7` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fbc93a0e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index eeeebba00..0f014a2da 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d934a5f7` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ fbc93a0e` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 70cc3f288..d59b82900 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d934a5f7` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fbc93a0e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f2bf2f854..e00fc7073 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d934a5f7` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fbc93a0e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c7e5865e9..d6f31be06 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d934a5f7` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fbc93a0e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ebace86a5..a3d1173e6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d934a5f7` 验证存在 +- 源码校验：以上路径均已在 `main @ fbc93a0e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index fdde0a1fb..97fe4a2d2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d934a5f7`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fbc93a0e`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from bbb3ae01 \ - --to d934a5f7 \ + --from d934a5f7 \ + --to fbc93a0e \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 97de75803..fdf90c024 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213"] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742"] confidence: high --- @@ -58,6 +58,11 @@ confidence: high - H3 online FP8 当前不得与 layerwise offload 组合：offload 产生的 weight stride 会被 Cutlass FP8 kernel 拒绝。变更量化覆盖范围时，除逐层命中/排除与加载转换测试外， 必须同时保护 joint video/audio 质量；peak memory 只作为同 case report，不是稳定上界。 +- The two-GPU H100-80GB full-model FP8 quality test must apply `text_encoder_tp_size=2` in the + shared eager, TP2, VAE-tiling kwargs used by both the fused BF16 baseline and transformer-FP8 + candidate. It prevents baseline initialization OOM by sharding the colocated text encoder; + it is test-only resource-topology evidence, not a runtime behavior or cross-hardware claim. + ^[PR #6742] - 音频加载优先使用 torchaudio；当 TorchCodec/torchaudio 在 CPU-only aarch64 环境不可用 时，`reference_video.load_audio_file` 回退到 soundfile，再对 libsndfile 不支持的格式 通过 ffmpeg 转 WAV。该回退保持 `(channels, samples)` float32 与原始 sample rate 合同。 From 4005872a42e491950971b4ab90e4df0ab80fccdb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 02:41:32 +0800 Subject: [PATCH 365/465] knowledge(vllm-omni): sync bfe8967c (PR #5917) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/step-audio2/_index.md | 6 +++--- .../vllm-omni/models/step-audio2/architecture.md | 15 +++++++++------ 11 files changed, 24 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2cfd5af89..62a919fd5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d934a5f7b8829b970b7e642398dcd9e14038e4d1 + previous_audited_sha: fbc93a0e08e44bd92d68c8c12617642f47448990 audited_ref: main - audited_sha: fbc93a0e08e44bd92d68c8c12617642f47448990 + audited_sha: bfe8967ca29bf4253a30177f053759f0b6cab158 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 983b079de..326374336 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -fbc93a0e08e44bd92d68c8c12617642f47448990`. The canonical machine baseline +bfe8967ca29bf4253a30177f053759f0b6cab158`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 069e5ac3e..104ac984c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fbc93a0e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bfe8967c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0f014a2da..51a0a645f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fbc93a0e` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ bfe8967c` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d59b82900..c04a318ba 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fbc93a0e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bfe8967c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e00fc7073..7ff75875f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fbc93a0e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bfe8967c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d6f31be06..ad2966673 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fbc93a0e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bfe8967c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a3d1173e6..19effd9dc 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fbc93a0e` 验证存在 +- 源码校验：以上路径均已在 `main @ bfe8967c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 97fe4a2d2..9051ac5a8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fbc93a0e`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bfe8967c`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from d934a5f7 \ - --to fbc93a0e \ + --from fbc93a0e \ + --to bfe8967c \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/step-audio2/_index.md b/knowledge/repos/vllm-omni/models/step-audio2/_index.md index 8f1a7d358..032facab2 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/_index.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/_index.md @@ -1,15 +1,15 @@ --- title: "Step-Audio2（音频 token 内嵌 LM 词表的语音对话）" created: 2026-07-21 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: ["PR #5067", "PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py, vllm_omni/platforms/npu/models/step_audio2_token2wav.py, vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py] +sources: ["PR #5067", "PR #5638", "PR #5869", "PR #5917", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py, vllm_omni/platforms/npu/models/step_audio2_token2wav.py, vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py] --- # Step-Audio2 -以下事实在 `main @ 12a5f6fb` 复核。 +以下事实在 `main @ bfe8967c` 复核。 ## 名称与范围 diff --git a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md index 1ef710ff4..756f07ae9 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md @@ -1,15 +1,15 @@ --- title: "Step-Audio2 架构" created: 2026-07-21 -updated: 2026-09-02 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models] -sources: ["PR #5067", "PR #5638", "PR #5869", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py] +sources: ["PR #5067", "PR #5638", "PR #5869", "PR #5917", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py, tests/model_executor/models/step_audio2/test_step_audio2_token2wav_async_chunk.py] --- # Step-Audio2 架构 -事实在 `main @ 12a5f6fb` 复核;变体/入口速览见 [index](_index.md)， +事实在 `main @ bfe8967c` 复核;变体/入口速览见 [index](_index.md)， 设备边界与同步验收见 [rules](rules.md)。 ## 模型专有部分与共享模块的边界 @@ -61,9 +61,12 @@ sources: ["PR #5067", "PR #5638", "PR #5869", vllm_omni/model_executor/models/co 3 个 lookahead 被刻意重发给下一块（conformer 编码器需要未来 token,内部 缓存）;末块发送全部剩余 token,纯文本完成发空 EOF 载荷。与 higgs/mimo 的 左上下文滑窗是不同机制。 -3. **跨家族陷阱**：本家族 payload meta 的 `left_context_size` 被复用为 - "是否末块"布尔（0/1）,**不是重叠帧数**——读 connector 元数据的共享代码 - 勿按 higgs/mimo 语义解释。 +3. **跨家族陷阱**：本家族由 runtime additional-information 条目的嵌套 + `meta.left_context_size` 识别 async-chunk；只要该值存在（包括非末块的 + `0`）就进入 token2wav 流式路径。它复用为"是否末块"布尔（`0/1`）， + **不是重叠帧数**；`1` 传给 core 作为 `last_chunk`，而读 connector 元数据的 + 共享代码不得按 higgs/mimo 语义解释。空 EOF 仍返回一个零 waveform chunk，并 + 不因本次 metadata 路由修复改变。^[PR #5917] 4. token2wav 流式:`_StreamState`（mel cache 160 ms、source cache 3840 样本、 estimator cache 窗 100）,块缝用 Hamming `fade_in_out` 平滑。 From 81ac6201c52776963284cc9eb1a5ca8108920473 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 02:49:48 +0800 Subject: [PATCH 366/465] knowledge(vllm-omni): sync 457c6e27 (PR #6745) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 16 ++++++++++------ .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 22 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 62a919fd5..03b9cb670 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fbc93a0e08e44bd92d68c8c12617642f47448990 + previous_audited_sha: bfe8967ca29bf4253a30177f053759f0b6cab158 audited_ref: main - audited_sha: bfe8967ca29bf4253a30177f053759f0b6cab158 + audited_sha: 457c6e27938595fac0c71ed7fe292b436b6eeb57 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 326374336..f44e08d76 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bfe8967ca29bf4253a30177f053759f0b6cab158`. The canonical machine baseline +457c6e27938595fac0c71ed7fe292b436b6eeb57`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 07b627fb3..702828e46 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", "PR #5464", "PR #6730", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464", "PR #6730", "PR #6745", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742"] --- # vLLM-Omni CI @@ -13,12 +13,16 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie - 处理 vLLM-Omni 的 L2/L4、模型测试配置或 CI 特有问题。 -## Historical MiniCPM-o 4.5 unit-test revert +## MiniCPM-o 4.5 full-payload regression coverage -- PR #6730 reverted the two PR #5464 CPU assertions after CI failure. They are not current - coverage and do not evidence a runtime, offline-inference, online-serving, or transport - implementation change. Reintroduce comparable coverage only with a passing CI-backed change. - ^[PR #5464] ^[PR #6730] +- PR #6730 had reverted the two PR #5464 CPU assertions after CI failure. PR #6745 reports + re-landing them after Buildkite build 14300 passed: with `async_chunk=False`, deployment merge + selects Talker’s `tts2code2wav_full_payload`; and a terminal `pooling_output=None` full payload returns + an empty terminal packet (`code_flat_numel=0`, `left_context_size=0`, `last_chunk=True`, and + `finished=True`) rather than `None`, releasing the downstream consumer wait gate. +- This commit modifies only those two tests. It refreshes regression coverage for existing + behavior; it does not itself change runtime, offline-inference, online-serving, or transport + implementation. ^[PR #5464] ^[PR #6730] ^[PR #6745] ## LTX-2 FP8 quality-gate recipe boundary diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 104ac984c..9c025354d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bfe8967c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 457c6e27` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 51a0a645f..47600ea85 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bfe8967c` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 457c6e27` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c04a318ba..22e5f834d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bfe8967c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 457c6e27` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7ff75875f..548bfc217 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bfe8967c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 457c6e27` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ad2966673..99b103ff7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bfe8967c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 457c6e27` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 19effd9dc..e4224bccf 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bfe8967c` 验证存在 +- 源码校验：以上路径均已在 `main @ 457c6e27` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9051ac5a8..ea75ce30e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bfe8967c`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 457c6e27`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from fbc93a0e \ - --to bfe8967c \ + --from bfe8967c \ + --to 457c6e27 \ --repo \ --mode report-only ``` From caaafd7a38c339a777ab8f630d70e8d89993bb01 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 02:58:25 +0800 Subject: [PATCH 367/465] knowledge(vllm-omni): sync f66f0563 (PR #6543) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/moss-tts/_index.md | 12 +++++++++-- .../repos/vllm-omni/models/moss-tts/rules.md | 21 +++++++++++++++++-- 11 files changed, 41 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 03b9cb670..8ca851174 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bfe8967ca29bf4253a30177f053759f0b6cab158 + previous_audited_sha: 457c6e27938595fac0c71ed7fe292b436b6eeb57 audited_ref: main - audited_sha: 457c6e27938595fac0c71ed7fe292b436b6eeb57 + audited_sha: f66f05633cfcf435b3de2966f7f3987646577c82 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f44e08d76..9e0c5a6df 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -457c6e27938595fac0c71ed7fe292b436b6eeb57`. The canonical machine baseline +f66f05633cfcf435b3de2966f7f3987646577c82`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9c025354d..d27cf3e3a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 457c6e27` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f66f0563` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 47600ea85..bb27c3833 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 457c6e27` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ f66f0563` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 22e5f834d..fdf9a25a0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 457c6e27` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f66f0563` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 548bfc217..bf895971d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 457c6e27` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f66f0563` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 99b103ff7..dc74623a5 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 457c6e27` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f66f0563` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e4224bccf..04e99840b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 457c6e27` 验证存在 +- 源码校验：以上路径均已在 `main @ f66f0563` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ea75ce30e..1bff6a0c5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 457c6e27`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f66f0563`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from bfe8967c \ - --to 457c6e27 \ + --from 457c6e27 \ + --to f66f0563 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/moss-tts/_index.md b/knowledge/repos/vllm-omni/models/moss-tts/_index.md index c8d08ad80..baa138c25 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/_index.md @@ -4,12 +4,12 @@ created: 2026-07-21 updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: ["PR #5635", "PR #6664", vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/model_executor/models/moss_tts/, vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py, vllm_omni/model_executor/models/moss_tts_nano/, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/] +sources: ["PR #5635", "PR #6664", "PR #6543", vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py, vllm_omni/model_executor/models/moss_tts/, vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py, vllm_omni/model_executor/models/moss_tts_nano/, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/] --- # MOSS-TTS 家族 -以下事实在 `main @ be335a86` 复核。 +以下事实在 `main @ f66f0563` 复核。 ## 名称与范围 @@ -42,6 +42,13 @@ sources: ["PR #5635", "PR #6664", vllm_omni/model_executor/models/registry.py, v - 依赖共享模块：[Config 组件](../../components/configuration/architecture.md)、 `utils/speaker_cache`、serving 适配 `entrypoints/openai/tts_adapters/moss_tts.py`。 +- shared speech path 在把 `request.seed` 应用到 `SamplingParams` 前调用 adapter `build()`， + 因此 MOSS adapter 必须自行优先使用显式 request seed（包括 `0`），仅在请求省略时回退 + stage-0 默认 seed。这个共享 + adapter contract 同时覆盖 full 与 Nano；端到端效果不是全族可复现性：Nano 消费该字段， + Local 早已经由 `tts_local_seed` 和逐请求 generator 接收 request seed，Delay/Realtime 仍缺 + request-scoped RNG plumbing。Nano 仍使用 global torch RNG，shipping `max_num_seqs: 4` 下 + 并发异 seed 请求并不隔离。^[PR #6543] ## 目录内容 @@ -49,6 +56,7 @@ sources: ["PR #5635", "PR #6664", vllm_omni/model_executor/models/registry.py, v |---|---|---| | delay 生命周期、伪文本 logits、双代 tokenizer | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | | codec v1/v2 选择、projection/checkpoint topology | [rules](rules.md#direct-代码快速入口) | loader 门禁与测试缺口 | +| online request `seed`、Nano/Local/Delay/Realtime 的可复现性或并发 | 同页 `MOSSTTS-3a` | adapter precedence 与各变体 RNG 边界 | ## 配置与 checkpoint 差异（8 份 deploy 速查） diff --git a/knowledge/repos/vllm-omni/models/moss-tts/rules.md b/knowledge/repos/vllm-omni/models/moss-tts/rules.md index 940616a92..8cdd23f90 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/rules.md @@ -1,10 +1,10 @@ --- title: "MOSS-TTS 规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, "PR #6241", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py"] +sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, "PR #6241", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py", "PR #6543", vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, tests/entrypoints/openai_api/test_tts_adapter.py] confidence: high --- @@ -18,6 +18,7 @@ confidence: high |---|---|---| | codec v1/v2、`number_channels`、missing weights | `MOSSTTS-1a` | `modeling_moss_tts_codec.py::MossTTSCodecDecoder._build_codec` → Stage-1 weight loader | | `_ProjectedTransformer`、`in_proj`/`out_proj`、Identity | `MOSSTTS-1a` | `audio_tokenizer.py::_ProjectedTransformer` → codec checkpoint parameter names | +| online request `seed`、adapter additional information、Nano/Local/Delay/Realtime reproducibility | `MOSSTTS-3a` | `tts_adapters/moss_tts.py::_MossTTSAdapterBase.build` → variant model seed/RNG consumer | ## MOSSTTS-1a — codec 代际与 module topology 必须由 checkpoint 原始结构决定 @@ -55,3 +56,19 @@ confidence: high - 禁止：在每个 depth step 内重复执行 `arange`/`einsum`/`cos`/`sin`，在 captured body 中构造或替换 cache，或用环境变量把这些直接实现伪装成可选路径；不得声称 compact sampler 与宽度 1024 的 multinomial 在 seed 或 bit 上等价，也不得忽略 top-k 边界 ties 的有限精度差异。 - 验收：RoPE 测试与参考公式数值一致，较短 slice 保持 cache storage；capture 前确认完整 cache 已物化且执行体只使用切片。采样测试断言结果属于 retained top-k、极小 `top_p` 选择 argmax，并在无 ties 时核对分布语义；质量或回归记录必须明确 compact width 改变 RNG 映射后的非 seed/bit-equivalence 边界。 ^[PR #6241] +## MOSSTTS-3a — 请求 seed 先在 adapter 构造 additional information，再判定模型 consumer + +- 触发：修改 MOSS-TTS OpenAI adapter、speech request seed propagation、`additional_information`， + 或宣称任一 MOSS 变体的 request-seed 可复现性。 +- 强制：`_MossTTSAdapterBase.build()` 在 shared speech path 将 request seed 写入 + `SamplingParams` 之前，优先把显式 `request.seed`（包括 `0`）写为 `tts_params["seed"]`； + request 未提供 seed 时才回退 stage-0 sampling default。分别追踪 consumer：Nano 从该 + additional information 读取 seed；Local 经 `tts_local_seed` 和逐请求 generator；Delay 和 + Realtime 需要各自的 request-scoped RNG/generator plumbing 才能获得模型侧效果。 +- 禁止：用 truthiness 丢弃 `seed=0`；把 adapter 的 shared contract test 当作 full-family + end-to-end reproducibility 证据；把 Local 的既有路径说成此 adapter 修复；或忽略 Nano 以 + global torch RNG 在并发请求间互相重置的限制。 +- 验收：对 `MossTTSAdapter` 与 `MossTTSNanoAdapter` 都覆盖 `seed=0`、非零 request seed + 覆盖 stage default，且 request 缺省时回退 default；模型级结论分别验证 consumer。Nano 的 + 并发异 seed 隔离需要在 shipping `max_num_seqs=4` 下单独以 per-request generator 验证，不能 + 由 adapter CPU contract 或单请求结果推出。^[PR #6543] From 3202f29daa868884f1fa0a33cf819e4e467dcbbd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:00:58 +0800 Subject: [PATCH 368/465] knowledge(vllm-omni): sync 55a226dc (PR #6747) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../model-executor/rules-image-task-envelope.md | 13 ++++++++++--- .../vllm-omni/components/model-executor/rules.md | 4 ++-- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 24 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8ca851174..821533a93 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 457c6e27938595fac0c71ed7fe292b436b6eeb57 + previous_audited_sha: f66f05633cfcf435b3de2966f7f3987646577c82 audited_ref: main - audited_sha: f66f05633cfcf435b3de2966f7f3987646577c82 + audited_sha: 55a226dcf1699cc99b068bf0939ab34f4f120d54 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9e0c5a6df..9348f0f6f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f66f05633cfcf435b3de2966f7f3987646577c82`. The canonical machine baseline +55a226dcf1699cc99b068bf0939ab34f4f120d54`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d27cf3e3a..76b72015e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f66f0563` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 55a226dc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index bb27c3833..5f478824d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f66f0563` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 55a226dc` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index fdf9a25a0..a7b6e1060 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f66f0563` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 55a226dc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index bf895971d..b39dc6985 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f66f0563` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 55a226dc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md index 46c785ff5..875532c8f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-image-task-envelope.md @@ -1,16 +1,16 @@ --- title: "image task envelope 合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #6049", "PR #6076", vllm_omni/model_extras/, "PR #6070", "vllm_omni/model_extras/registry.py", "PR #5614", "PR #6353"] +sources: ["PR #6049", "PR #6076", "PR #6747", vllm_omni/model_extras/, "PR #6070", "vllm_omni/model_extras/registry.py", "PR #5614", "PR #6353", examples/offline_inference/text_to_video/text_to_video.py, examples/offline_inference/image_to_video/image_to_video.py, tests/examples/offline_inference/test_image_task_prompts.py] confidence: high --- # image task envelope 合同 -`image-task-envelope` 审查组的 `EXEC-6a`–`EXEC-6b`：shared image example 的 canonical envelope 与 `model_extras` 的模型专有参数声明。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 +`image-task-envelope` 审查组的 `EXEC-6a`–`EXEC-6d`：shared image/video example 的 canonical envelope、输出 consumer 与 `model_extras` 的模型专有参数声明。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 ## EXEC-6a — shared image example 先建 canonical envelope，model-extra 只做特化变换 @@ -32,6 +32,13 @@ confidence: high - 禁止：把模型专有选项散落到共享 runner 的通用参数或绕过 registry；按每帧当前最小值推断范围；把已是 `[0, 1]` 的模型输出再次按 `[-1, 1]` 映射。 - 验收：registry 测试断言目标模型的 extra-body 参数和 `zero_to_one` 范围、普通 pipeline 的默认范围；覆盖单帧与 list-valued 视频的明确范围、混合正负值的统一转换，以及最终视频导出结果。 ^[PR #6076] +## EXEC-6d — shared offline video consumer 必须接受语义 video 与 legacy image 标签 + +- 触发：修改 shared T2V/I2V offline example 的 `OmniRequestOutput.final_output_type` 校验或视频帧消费。 +- 强制：consumer 接受语义 `video` 和兼容旧 pipeline 的 `image`；继续从历史 `images` 字段取得视频帧，并对其他输出类型抛出明确错误。 +- 禁止：将 `images` frame carrier 改写为生成器、formatter、post-process 或 encoder 的输出类型合同；不得因兼容 `image` 而静默接收 `audio` 等无关 modality。 +- 验收：text-to-video 与 image-to-video 都覆盖 `video`、`image` 通过及 `audio` 拒绝；这些测试只证明 consumer contract，不证明帧生成、后处理或编码行为。^[PR #6747] + 相关执行流见 [model-executor architecture](architecture.md)；跨 stage 合同见 [bridge/batch 规则](rules-bridge-batch.md)。 ## EXEC-6c — model_extras resolver 必须承载模型专有版本策略 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index a46cc9d47..dd6cfa8ad 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -27,7 +27,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | upstream runner/output API drift、`num_nans`、sample-budget 或 quant loader helper moved | `async-output` + output-contract | live upstream runner/output dataclasses → `gpu_{ar,generation}_model_runner.py` / `outputs/output_processor.py` → diffusion quant configs；moved symbol 必须用目标 release 的 canonical import，而非保留旧 fallback ^[PR #6606] | | Talker-MTP FULL graph、平台 capture 能力、显式 opt-out | `mtp-graph`：`EXEC-4c` | `platforms/interface.py::supports_talker_mtp_graph_capture` → platform override → model `talker_mtp_graph_safe` → `OmniGPUModelRunner._init_talker_mtp` | -| `model_extras`、shared T2I/I2V example、canonical prompt envelope | `image-task-envelope`：`EXEC-6a` | `examples/offline_inference/{text_to_image/text_to_image.py,image_to_video/image_to_video.py}` → `model_extras/registry.py::{build_text_to_image_prompt,build_image_to_video_prompt}` → model pipeline validation | +| `model_extras`、shared T2I/T2V/I2V example、canonical prompt envelope 或 video output consumer | `image-task-envelope`：`EXEC-6a`, `EXEC-6d` | `examples/offline_inference/{text_to_image/text_to_image.py,text_to_video/text_to_video.py,image_to_video/image_to_video.py}` → `model_extras/registry.py::{build_text_to_image_prompt,build_image_to_video_prompt}` → model pipeline validation | | CosyVoice3 non-module TensorRT CFM、stream/event ownership、raw-pointer buffer lifetime 或 context-pool reuse | [CosyVoice3 TensorRT handoff](../../models/cosyvoice3/rules.md)：`COSYVOICE3-1a` | `code2wav_core/cfm.py::ConditionalCFM.forward_estimator` → `flow_estimator_trt.py::TrtContextWrapper` | | Qwen3-TTS adaptive chunk、EWMA target、all-frame emit、controller cleanup | [Qwen3-TTS adaptive ramp](../../models/qwen3-tts/rules.md#q3tts-3e-adaptive-ramp-是-host-side每段-opt-in-控制器不是-cuda-graph-计划) | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `chunk_size_utils.py::{AdaptiveChunkController,compute_adaptive_emit}` → Code2Wav | @@ -40,7 +40,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`–`EXEC-2c`，见 [loader 合同](rules-loader-contract.md) | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a`, `EXEC-5b` | | `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | -| `image-task-envelope` | shared image task example 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6b`，见 [image task envelope 合同](rules-image-task-envelope.md) | +| `image-task-envelope` | shared image/video task example、video output consumer 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6d`，见 [image task envelope 合同](rules-image-task-envelope.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | | `output-contract` | Omni 输出类型与字段/复制合同 | `EXEC-7a`–`EXEC-7b`，见 [输出类型合同](rules-output-contract.md) | | `runtime-hot-paths` | 采样循环不变量、固定输入缓存、AR 音频侧路、codec 帧账本与 CosyVoice3 typed handoff | `EXEC-8a`, `EXEC-9a`, `EXEC-11a`–`EXEC-11h`，见 [运行时热路径合同](rules-runtime-hot-paths.md) | diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index dc74623a5..e183f6b4c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f66f0563` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 55a226dc` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 04e99840b..4829bf70d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f66f0563` 验证存在 +- 源码校验：以上路径均已在 `main @ 55a226dc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1bff6a0c5..f262a9607 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f66f0563`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 55a226dc`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 457c6e27 \ - --to f66f0563 \ + --from f66f0563 \ + --to 55a226dc \ --repo \ --mode report-only ``` From ef2e8ddfb7ef67edc92ee669cfbd5d44ba566d7d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:05:35 +0800 Subject: [PATCH 369/465] knowledge(vllm-omni): sync 143ccc5a (PR #4322) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/model-executor/rules.md | 3 ++- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/qwen-omni/_index.md | 6 +++--- knowledge/repos/vllm-omni/models/qwen-omni/rules.md | 11 +++++++++-- 12 files changed, 26 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 821533a93..0278af7fd 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f66f05633cfcf435b3de2966f7f3987646577c82 + previous_audited_sha: 55a226dcf1699cc99b068bf0939ab34f4f120d54 audited_ref: main - audited_sha: 55a226dcf1699cc99b068bf0939ab34f4f120d54 + audited_sha: 143ccc5a2ff4f95360f1214632eeffc10afbd5bb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9348f0f6f..92332a760 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -55a226dcf1699cc99b068bf0939ab34f4f120d54`. The canonical machine baseline +143ccc5a2ff4f95360f1214632eeffc10afbd5bb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 76b72015e..0ae1240b8 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 55a226dc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 143ccc5a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5f478824d..82fa41c12 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 55a226dc` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 143ccc5a` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a7b6e1060..d4d817249 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 55a226dc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 143ccc5a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b39dc6985..f9fac4e3b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 55a226dc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 143ccc5a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index dd6cfa8ad..9b88ea912 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149", "PR #4322"] --- # Model Executor 规则 @@ -30,6 +30,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `model_extras`、shared T2I/T2V/I2V example、canonical prompt envelope 或 video output consumer | `image-task-envelope`：`EXEC-6a`, `EXEC-6d` | `examples/offline_inference/{text_to_image/text_to_image.py,text_to_video/text_to_video.py,image_to_video/image_to_video.py}` → `model_extras/registry.py::{build_text_to_image_prompt,build_image_to_video_prompt}` → model pipeline validation | | CosyVoice3 non-module TensorRT CFM、stream/event ownership、raw-pointer buffer lifetime 或 context-pool reuse | [CosyVoice3 TensorRT handoff](../../models/cosyvoice3/rules.md)：`COSYVOICE3-1a` | `code2wav_core/cfm.py::ConditionalCFM.forward_estimator` → `flow_estimator_trt.py::TrtContextWrapper` | | Qwen3-TTS adaptive chunk、EWMA target、all-frame emit、controller cleanup | [Qwen3-TTS adaptive ramp](../../models/qwen3-tts/rules.md#q3tts-3e-adaptive-ramp-是-host-side每段-opt-in-控制器不是-cuda-graph-计划) | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `chunk_size_utils.py::{AdaptiveChunkController,compute_adaptive_emit}` → Code2Wav | +| Qwen3-Omni audio encoder 的 head/TP divisibility | [Qwen-Omni audio-encoder TP rule](../../models/qwen-omni/rules.md#qomni-1d-audio-encoder-tp-必须按-head-divisibility-局部回退) | `qwen3_omni_moe_thinker.py::{Qwen3OmniMoeAudioAttention,Qwen3OmniMoeAudioEncoderLayer}`；只影响该 encoder layer 的 QKV/output/FFN TP fallback ^[PR #4322] | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e183f6b4c..cd5459334 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 55a226dc` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 143ccc5a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4829bf70d..237b6a340 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 55a226dc` 验证存在 +- 源码校验：以上路径均已在 `main @ 143ccc5a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f262a9607..5f8cfec41 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 55a226dc`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 143ccc5a`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from f66f0563 \ - --to 55a226dc \ + --from 55a226dc \ + --to 143ccc5a \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index 1cb0255fc..4b9a8b3bf 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -1,10 +1,10 @@ --- title: "Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS）" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/quantization/component_config.py, docs/design/qwen3_omni_tts_performance_optimization.md] +sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/quantization/component_config.py, docs/design/qwen3_omni_tts_performance_optimization.md] --- # Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS） @@ -53,4 +53,4 @@ sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", vllm_omni/ | 遇到什么 | 查看哪里 | |---|---| | stage 拓扑、代际差异与官方性能优化结论 | [architecture](architecture.md) | -| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary、AWQ/compressed-tensors 名称映射或固定种子音频回归 | [Qwen-Omni rules](rules.md) | +| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary、AWQ/compressed-tensors 名称映射、固定种子音频回归，或 audio-encoder head/TP divisibility | [Qwen-Omni rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index e00dac0e2..eb957a51d 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -1,10 +1,10 @@ --- title: "Qwen-Omni 规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5687", "PR #6284", "PR #6449", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py] +sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py] confidence: high --- @@ -47,3 +47,10 @@ confidence: high 断言同一 `qwen3_omni_moe_thinker_only` pipeline、仅一个 Thinker stage 和 text output；同时 直接解析该 key，断言其对象身份为 registry 的静态 `PipelineConfig` 而非 resolver 结果。PR 只有这类配置单元测试，未提供真实 `vllm serve`、质量、性能或资源证据。^[PR #6284] + +## QOMNI-1d — audio encoder TP 必须按 head divisibility 局部回退 + +- 触发：修改 Qwen3-Omni audio encoder 的 attention/FFN parallel linear、`encoder_attention_heads`，或其 global tensor-parallel 初始化。 +- 强制：以 `encoder_attention_heads % global_tp_size` 判定该 audio-encoder layer 是否可分片。可整除时保持 global TP；不可整除时仅该 encoder layer 的 attention QKV、attention output projection、FC1 和 FC2 都以 `disable_tp=True` 构造，且 local attention head count 按 effective TP=1 计算。 +- 禁止：仅关闭 QKV 或仅修正 `num_local_heads` 而让 output/FFN 仍分片；把这项 fallback 扩大为整个 Qwen3-Omni 模型关闭 TP；或把 PR 的 TP1/2/4/8 手工音频请求、MI350x full-workload throughput 数字归因于该 fallback。 +- 验收：分别覆盖 heads 可被和不可被 global TP 整除的构造，断言四个 linear 的 `disable_tp` 一致、attention local-head count 正确，且 encoder 外的 Thinker/Talker TP 配置未被此条件改写。PR 仅报告手工请求与整体 workload 测量，未给出该局部回退的独立性能或质量证明。^[PR #4322] From b1962c822638476b76cad8420c1fae4c1eeda82b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:11:58 +0800 Subject: [PATCH 370/465] knowledge(vllm-omni): sync 08cdee8d (PR #6666) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minimax-h3/_index.md | 5 ++++- .../vllm-omni/models/minimax-h3/rules-cache-task.md | 13 +++++++++++-- .../repos/vllm-omni/models/minimax-h3/rules.md | 1 + 12 files changed, 28 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0278af7fd..8c5082549 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 55a226dcf1699cc99b068bf0939ab34f4f120d54 + previous_audited_sha: 143ccc5a2ff4f95360f1214632eeffc10afbd5bb audited_ref: main - audited_sha: 143ccc5a2ff4f95360f1214632eeffc10afbd5bb + audited_sha: 08cdee8dc4a2537cc90e660d82115338b35cb2b0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 92332a760..8b9973d03 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -143ccc5a2ff4f95360f1214632eeffc10afbd5bb`. The canonical machine baseline +08cdee8dc4a2537cc90e660d82115338b35cb2b0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0ae1240b8..d81760be1 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 143ccc5a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 08cdee8d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 82fa41c12..82ef5026f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 143ccc5a` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 08cdee8d` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d4d817249..ec8e33e82 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 143ccc5a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 08cdee8d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f9fac4e3b..50f23abaa 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 143ccc5a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 08cdee8d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index cd5459334..44de1c646 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 143ccc5a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 08cdee8d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 237b6a340..60e6542c8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 143ccc5a` 验证存在 +- 源码校验：以上路径均已在 `main @ 08cdee8d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5f8cfec41..105e280d2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 143ccc5a`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 08cdee8d`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 55a226dc \ - --to 143ccc5a \ + --from 143ccc5a \ + --to 08cdee8d \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index fdf90c024..296c71a34 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742"] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742", "PR #6666"] confidence: high --- @@ -96,6 +96,9 @@ confidence: high - LightX2V Turbo v1.0 仅支持 legacy dynamic LoRA；精确 artifact mapping、packed QKV/FC1 binding、active-request sampling 合同与 DLO-resident A/B sidecar 边界见 MMH3-2j；转换后的 mixed-rank native PEFT checkpoint 不在该支持声明内。 +- FlashGen native v1.0 同样经 legacy `DiffusionLoRAManager` dynamic path，但以 safetensors + `key_format=minimax-h3-native` 选择独立 parser；artifact、packing、schedule 与 offload gates + 见 MMH3-2m。NPU 子目录或作者报告均不构成平台、性能或端到端验证。^[PR #6666] - datacenter Blackwell 上 H3 可默认 dense BF16 TRTLLM；single-request producer-owned `PackedPaddingMetadata` 可替代 structural suffix mask，generic ragged/continuous batches仍按 cu_seqlens 隔离 documents。SAGE short-role 与 multi-request tail gate、mask/metadata fail-closed、 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md index 715363d6a..94fe92665 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md @@ -1,10 +1,10 @@ --- title: "MiniMax H3 缓存与任务生命周期规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", "PR #6550", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py] +sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", "PR #6550", "PR #6666", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/npu/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_native_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py] confidence: high --- @@ -170,6 +170,15 @@ Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期 - 禁止：因 backend 名称为 `FLASH_ATTN`、平台默认或容量大于 1 就假设可合批；在多 document `cu_seqlens` 不被 backend 消费时去掉 mask；将不同宽度的 audio rows 混入 video batch；让 request-scoped Cache-DiT hooks、DLO resident window 或多输出 latent 跨 state 复用。不得把 H3-local broadcast 夸大为通用 runner 防护：target 的 `_dit_any_rank_failed()` 无法取得 DiT group 而退回 local flag，且 group-device 选择仍有 backend mismatch 风险。 - 验收：CPU contract 覆盖 request/step helper parity、two-request packed document boundaries、video/audio split/reassembly、mixed-DiT/Ring/unsupported-backend fallback、abort-after-inflight-step，及三个明确 rejection；TeaCache extractor 继续以 omitted `num_requests=1` 走 request-mode path。真实权重只可在固定 checkpoint、seed、task、backend、topology 下比较 single vs packed outputs；并发 step run 的 packing composition 会随 arrival timing 改变，PR 的 4×H100 结果已显示 step rerun 和 request-vs-step 都非 bitwise 等价。有限 CPU、H100/B300 observations 和 workload-specific throughput 都不是所有 backend、small request、确定性或 multi-rank failure synchronization 的证明。^[PR #5810] +## MMH3-2m — FlashGen native LoRA 必须保持 artifact、schedule 与 legacy-manager 边界 + +- 触发：FlashGen native artifact loading、H3 dynamic LoRA classification/binding、adapter schedule、packed QKV/FC1，或与 task、step execution、CPU/layerwise/DLO/HWR offload 的组合。 +- 强制：只接受 exact v1.0 safetensors file；metadata 必须为 `key_format=minimax-h3-native`、`qkv_layout=grouped`、rank/alpha `64/64`、`tasks=t2va` 和五个 boundary 的四-interval `base_schedule`；target set 必须完整且精确（259 个），A/B pair、matrix shape、文件名或 metadata 任一不符均 fail closed。按 native module name 映射；grouped `qkv_proj` 先复用 H3 base-loader reorder 再拆 Q/K/V，`fc1` 拆 gate/up，二者复用 full-input A、以 slice-local B 绑定 packed layers。 +- 强制：artifact format 而非 running platform 选择 native loader；loader 读取 CPU safetensors、无 `torch_npu` dispatch，但最终 binding/execution 仍由 selected runtime platform 负责。adapter 仅在 nonzero-scale active request 时覆盖 checkpoint schedule；T2VA request mode 可省略 steps 并从 adapter 取 4 intervals，explicit step 必须为 4；step execution 必须显式给 4，因为 admission 早于 adapter schedule。native adapter 不得与 checkpoint-pinned `base_schedule` 或 Ref2VA 组合；真实 reload/eviction 必须清掉 reused client ID 的 native classification/schedule。 +- 强制：保持既有 legacy `DiffusionLoRAManager`，不引入 native-only manager、prefusion 或 multi-active composition。动态 A/B tensors 不参与 model-level CPU 或 standard layerwise weight lifecycle，因此二者拒绝；request-mode DLO 仅 stream base blocks、A/B 留 compute device。step execution 仍拒绝 DLO；HWR 的 `lora_path` eligibility 与 AllGather transport 未被本 PR 改变，不能据此宣称 HWR 或 LoRA-AllGather support。 +- 禁止：把 NPU directory/训练来源解释为 NPU-only 或已验证跨平台性能；把 request-mode omission 扩展到 step execution；把 on-disk adapter size 作为 rank-local HBM bound；或以 unit/mocked binding、作者的单一硬件报告声称真实 artifact E2E、quality、throughput 或 other topology evidence。 +- 验收：mock/synthetic artifact 覆盖 metadata/file/target/pair/shape/schedule rejection、grouped QKV 与 FC1 packing、legacy manager complete binding、active vs scale-zero、same ID reload/eviction、T2VA/Ref2VA、request/step count、offload gates 与 TP slice divisibility。PR 新增 native-LoRA unit contracts，未提交真实 release artifact 的 GPU/NPU E2E 或 independent quality/performance gate。^[PR #6666] + 共享 component quantization、checkpoint mapping 与 quality evidence 见 [Diffusion rules](../../components/diffusion/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index b0b70706b..d83225e45 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -37,6 +37,7 @@ confidence: high | TeaCache、FL2VA coefficients、0.17、combined/Ref2VA | [MMH3-2g](rules-cache-task.md#mmh3-2g-h3-teacache-只绑定-fl2va-校准与-request-state) | custom enabler → H3 extractor → module-resident hook state / per-generation reset | | distilled、DMD2、`base_schedule`、4-step、sigma boundary | [MMH3-2h](rules-cache-task.md#mmh3-2h-distilled-sigma-schedule-按-partition-所有且以区间计步) | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | | LightX2V Turbo、legacy dynamic LoRA、packed QKV/FC1、five sigma points、DLO resident A/B sidecars | [MMH3-2j](rules-cache-task.md#mmh3-2j-turbo-lora-只接受精确发布-artifact并与-h3-tasksampling-和-offload-lifecycle-共同门禁) | `lora.py::load_minimax_h3_turbo_lora` → pipeline hook/sampling validation → `DiffusionLoRAManager` | +| FlashGen native LoRA、grouped QKV、adapter sigma schedule、legacy manager binding | [MMH3-2m](rules-cache-task.md#mmh3-2m-flashgen-native-lora-必须保持-artifactschedule-与-legacy-manager-边界) | `npu/lora.py::load_minimax_h3_native_lora` → H3 pipeline active-adapter validation → unchanged `DiffusionLoRAManager` ^[PR #6666] | | step execution、continuous batching、multi-document packing、rank-0 prepare failure | [MMH3-2k](rules-cache-task.md#mmh3-2k-h3-step-execution-必须保持请求状态attention-文档和-rank-0-prepare-隔离) | `pipeline_minimax_h3.py::{prepare_encode,denoise_step,step_scheduler}` → `batched_packing.py` → runner | | DLO、TP-local、resident layers、allocator cache、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → `BoundedAllocatorCache`/`PinnedModuleStager` → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | From 91f2fb62e9eac0a6e185edbc7dc60aaf4fc33185 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:14:39 +0800 Subject: [PATCH 371/465] knowledge(vllm-omni): sync 5c47fd0b (PR #6748) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/qwen-omni/rules.md | 10 +++++++--- 10 files changed, 19 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8c5082549..3dcfd7e9d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 143ccc5a2ff4f95360f1214632eeffc10afbd5bb + previous_audited_sha: 08cdee8dc4a2537cc90e660d82115338b35cb2b0 audited_ref: main - audited_sha: 08cdee8dc4a2537cc90e660d82115338b35cb2b0 + audited_sha: 5c47fd0bc8a3e5d980cddfe5e052657777b5632f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8b9973d03..1654dad3f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -08cdee8dc4a2537cc90e660d82115338b35cb2b0`. The canonical machine baseline +5c47fd0bc8a3e5d980cddfe5e052657777b5632f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d81760be1..8dec47361 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 08cdee8d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5c47fd0b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 82ef5026f..aa4126841 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 08cdee8d` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 5c47fd0b` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ec8e33e82..8e1c4298d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 08cdee8d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5c47fd0b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 50f23abaa..958a6c34c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 08cdee8d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5c47fd0b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 44de1c646..201748e01 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 08cdee8d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5c47fd0b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 60e6542c8..ed0523668 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 08cdee8d` 验证存在 +- 源码校验：以上路径均已在 `main @ 5c47fd0b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 105e280d2..5c1f3b8e1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 08cdee8d`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5c47fd0b`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 143ccc5a \ - --to 08cdee8d \ + --from 08cdee8d \ + --to 5c47fd0b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index eb957a51d..fe7702bc7 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py] +sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py] confidence: high --- @@ -24,13 +24,17 @@ confidence: high - 强制：在统一 Qwen3-Omni wrapper 暴露 checkpoint 到最终嵌套模块路径的映射，并先对完整 quantization config 执行一次映射；`ComponentQuantizationConfig` 必须递归映射每个非空 component 和 default config。子 stage 检测到该映射后不得再次改写名称，但仍须合并自己的 - `packed_modules_mapping`，供量化 scheme 匹配 fused module。 + `packed_modules_mapping`，供量化 scheme 匹配 fused module。该嵌套 metadata 是 optional：outer-map-once + marker 已存在时，缺少 attribute 必须 no-op；存在且非空时才 merge，且不得清掉 outer marker。 - 禁止：只映射一个 component wrapper 而给它标记为已映射；让 Thinker/Talker/Code2Wav 各自 重复应用 stage mapper；因修复名称匹配而把 BF16 router/encoder 也纳入量化；或把该构造期 映射修复描述成 quantization kernel / inference hot path 变化。 - 验收：以真实 component-wrapped `CompressedTensorsConfig`（含 `None` component 与 default） 断言 `ignore` HF names 映射到最终路径；加载 AWQ checkpoint 时三 stage 都能完成 weight - loading，且不出现缺模块/参数或 routed-expert quantized-weight 属性错误。^[PR #5687] + loading，且不出现缺模块/参数或 routed-expert quantized-weight 属性错误。CPU regression 还必须覆盖 + missing quant config、outer-mapped Talker-style model 缺 `packed_modules_mapping` 的 no-op、及 present + mapping merge；这只是 mapping/metadata contract，不是 quant scheme、kernel、quality 或 performance + evidence。^[PR #5687] ^[PR #6748] ## QOMNI-1c — Instruct 的 thinker-only deploy 必须选择静态 pipeline key From 31f535f8e41377a8cae9b6c1fdebf6904ea6f85b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:17:26 +0800 Subject: [PATCH 372/465] knowledge(vllm-omni): sync bb5bf8fd (PR #6367) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 10 ++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 20 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3dcfd7e9d..67fa70036 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 08cdee8dc4a2537cc90e660d82115338b35cb2b0 + previous_audited_sha: 5c47fd0bc8a3e5d980cddfe5e052657777b5632f audited_ref: main - audited_sha: 5c47fd0bc8a3e5d980cddfe5e052657777b5632f + audited_sha: bb5bf8fd0af4dcd84496e16b24cb5ce6f4d7fb17 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1654dad3f..d130805d9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5c47fd0bc8a3e5d980cddfe5e052657777b5632f`. The canonical machine baseline +bb5bf8fd0af4dcd84496e16b24cb5ce6f4d7fb17`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8dec47361..be58f5eaa 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5c47fd0b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ bb5bf8fd` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index aa4126841..359622e35 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5c47fd0b` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ bb5bf8fd` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8e1c4298d..06d914a97 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5c47fd0b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ bb5bf8fd` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 958a6c34c..bc30e13dc 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5c47fd0b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ bb5bf8fd` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 201748e01..a9bf2f0ea 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5c47fd0b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ bb5bf8fd` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ed0523668..6e15fb7c4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5c47fd0b` 验证存在 +- 源码校验：以上路径均已在 `main @ bb5bf8fd` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 7634ee65f..4249c43a9 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5221", "Issue #4855", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, tests/engine/test_orchestrator_event_driven.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py, "PR #6189", tests/entrypoints/test_async_omni_diffusion_config.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5221", "Issue #4855", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, tests/engine/test_orchestrator_event_driven.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py, "PR #6189", tests/entrypoints/test_async_omni_diffusion_config.py, "PR #6367", vllm_omni/outputs/output_processor.py, tests/engine/test_async_omni_engine_abort_ack.py, tests/entrypoints/test_async_omni_pause_sleep_routing.py] confidence: high --- @@ -160,6 +160,12 @@ confidence: high timeout；其余 RPC 仍走 collective path。需要 frontend cleanup 的 abort 必须关联 result，等待 orchestrator 完成 stage abort、binding release 和 request cleanup 后才移除 frontend state，失败或 timeout 必须保留 state 并传播。 +- 强制：sleep 先关闭 admission，再等待所有 in-flight `_admitting` submissions drained 后才 offload；P0 + 先 reset MM cache，stage tag 保留 per-stage scope。wake 不重新开启 AR admission，仍只由 + `resume_generation()` 解除。AR abort 必须把 cumulative terminal-prefix token 由 output processor + 传至 stage pool/orchestrator 并 ACK 回 async queue；先物理 abort，再 commit output/request state，且 + state 延迟到 generator 消费后清理。只移除最后一个 child，只有最后 terminal output 收敛；control RPC + exceptions 必须传播。diffusion abort 路径未因本 PR 改写。 - 禁止：以单一 frontend flag 代替 AR scheduler 控制；在 sleep RPC 后才阻止 `generate()`；让 wake 隐式 resume；因已 paused 跳过不同 stage scope 或 cache reset；将任意同名 `*_async` helper 绕过 collective timeout；在 abort ack 前 pop request state，或将 abort failure 当成功。 @@ -168,7 +174,7 @@ confidence: high `generate()` 前显式调用 `resume_generation()`，而 diffusion-only sleep → wake 可恢复 admission；重复/定向 pause 仍调用 scheduler 与 cache clear，非法 stage ID 产生明确错误，fast path 仅命中四个方法且 timeout 生效；abort success 后才清理 frontend state，orchestrator error 与 timeout 时 state 保留。 - ^[PR #6084] ^[PR #6581] + 本 PR 的测试/PR body 没有独立 GPU performance 或全量 diffusion-abort evidence。^[PR #6084] ^[PR #6581] ^[PR #6367] ### SERV-5n — remote replica attach/detach 必须按本次 membership generation 判定 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5c1f3b8e1..1c151b543 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5c47fd0b`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ bb5bf8fd`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 08cdee8d \ - --to 5c47fd0b \ + --from 5c47fd0b \ + --to bb5bf8fd \ --repo \ --mode report-only ``` From abc3124905c82b685360c733feae94d56b9feb27 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:22:46 +0800 Subject: [PATCH 373/465] knowledge(vllm-omni): sync d9980c75 (PR #6410) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/rules.md | 8 ++++---- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 4 +++- 11 files changed, 19 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 67fa70036..aeb44ef5c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5c47fd0bc8a3e5d980cddfe5e052657777b5632f + previous_audited_sha: bb5bf8fd0af4dcd84496e16b24cb5ce6f4d7fb17 audited_ref: main - audited_sha: bb5bf8fd0af4dcd84496e16b24cb5ce6f4d7fb17 + audited_sha: d9980c75e8fe68ba2e6aa84360f109182a75023b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d130805d9..7a59e4b61 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -bb5bf8fd0af4dcd84496e16b24cb5ce6f4d7fb17`. The canonical machine baseline +d9980c75e8fe68ba2e6aa84360f109182a75023b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index be58f5eaa..dd88b8e78 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ bb5bf8fd` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d9980c75` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 359622e35..fbe59fac2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ bb5bf8fd` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ d9980c75` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 22689eeba..a9b41089b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", "PR #5831", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150"] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", "PR #5831", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150", "PR #6410", tests/diffusion/layers/test_activation.py, tests/diffusion/layers/test_fused_qk_norm_rope_npu.py] confidence: high --- @@ -146,9 +146,9 @@ async output readiness、per-worker result channel、shutdown 与 constructor cl ### DIFF-1v — diffusion packed SwiGLU 必须保持平台分派与精度边界 - 触发：共享 diffusion MLP 将 `F.silu(gate) * up` 替换为 packed SwiGLU fused activation，或新增平台专用 activation dispatch。 -- 强制：输入最后一维必须保持 `[gate, up]` packed 合同，输出形状为输入去半后的最后一维，并保持输入的 device/dtype。通过本地 `CustomOp` 和 `current_omni_platform` 分派；CUDA 设备才调用 `torch.ops._C.silu_and_mul`，NPU/XPU 保持 native fallback，其他平台必须有独立且已注册的实现。不会实际使用 fused op 的平台不得在构造时 eager resolve 该 op，尤其是 MUSA 应延迟解析。 -- 禁止：依赖 vLLM 通用 `SiluAndMul` 代替 diffusion 本地合同；把 `CustomOp` 存在误认为 NPU 已使用 `torch_npu.npu_swiglu`；让 MUSA 构造因缺少 `_C.silu_and_mul` 失败；把 fused kernel 的输出宣称为必然 byte-identical，忽略不同舍入边界。 -- 验收：CUDA 上以 native reference 对照 fused 输出的 shape、dtype、device 和数值，并 profile 确认每个目标 MLP 使用 fused kernel；NPU/XPU 断言 native fallback，MUSA 覆盖无 op 注册时的构造与 forward，若启用 NPU fused path 则单独验证 `torch_npu.npu_swiglu`。固定 checkpoint、输入、seed 和步数比较 H3 输出时使用明确数值容差，不能以本 PR 的 byte-identical 报告替代跨平台证据。^[PR #6283] +- 强制：输入最后一维必须保持 `[gate, up]` packed 合同，输出形状为输入去半后的最后一维，并保持输入的 device/dtype。通过本地 `CustomOp` 和 `current_omni_platform` 分派；CUDA 调用 `torch.ops._C.silu_and_mul`，NPU 调用 `torch_npu.npu_swiglu(..., dim=-1)`，XPU 保持 native fallback，其他平台必须有独立且已注册的实现。不会实际使用 fused op 的平台不得在构造时 eager resolve 该 op，尤其是 MUSA 应延迟解析。 +- 禁止：依赖 vLLM 通用 `SiluAndMul` 代替 diffusion 本地合同；把 `CustomOp` 注册本身当作目标设备实际命中证明；让 MUSA 构造因缺少 `_C.silu_and_mul` 失败；把 fused kernel 的输出宣称为必然 byte-identical，忽略不同舍入边界。 +- 验收：CUDA 与目标 NPU 分别以 native reference 对照 fused 输出的 shape、dtype、device 和数值，并确认实际命中对应 kernel；XPU 断言 native fallback，MUSA 覆盖无 op 注册时的构造与 forward。固定 checkpoint、输入、seed 和步数比较 H3 输出时使用明确数值容差；NPU 路径只是目标 kernel contract，不产生通用性能或跨平台证据。^[PR #6283] ^[PR #6410] ### DIFF-1w — Triton 融合 kernel 必须有设备能力与连续布局门禁 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 06d914a97..514042b05 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ bb5bf8fd` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d9980c75` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index bc30e13dc..0ab71d59c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ bb5bf8fd` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d9980c75` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a9bf2f0ea..12c72ff53 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ bb5bf8fd` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d9980c75` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6e15fb7c4..f0c46d4eb 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ bb5bf8fd` 验证存在 +- 源码校验：以上路径均已在 `main @ d9980c75` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1c151b543..d127ead76 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ bb5bf8fd`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d9980c75`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 5c47fd0b \ - --to bb5bf8fd \ + --from bb5bf8fd \ + --to d9980c75 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index d83225e45..e47a6a3bd 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", "PR #6040", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526"] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", "PR #6040", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526", "PR #6410", vllm_omni/diffusion/models/minimax_h3/denoise_loop.py] confidence: high --- @@ -165,6 +165,7 @@ confidence: high - 强制：H3 先将 `MiniMaxH3Rope` 的 tiled frequency 输出交给 `_build_rope_table`，只从其前半段构造 cos/sin 并生成 BF16 packed table；该 table 必须在完整 packed sequence 上构造一次，经 `sp_prepare` 原样传给所有 DiT blocks 和 attention，TeaCache extractor 复用同一 helper。`rope_table=None` 的 token-refiner 路径继续只做 q/k RMSNorm；H3 的 128 维 head 只旋转前 96 维并保留后 32 维。共享算子边界以 [DIFF-1l](../../components/diffusion/rules.md#diff-1l-fused-qk-rmsnorm-与-packed-rope-必须共享布局并保留-eager-fallback) 为准。 - 禁止：把 H3 已复制的第二个 frequency half 当成独立 sin 输入，向 fused API 传 raw `rope_freqs`，或在每个 block、SP 分支和 TeaCache extractor 内重建不同 table；也不能把 token-refiner 的无 RoPE 路径误接到 fused table 调用。 - 验收：模型合同测试须断言完整 sequence 的 table shape 与 `[cos, sin]` 数值、SP 透传 identity、DiT q/k 的 96/128 partial rotation 和 32 维 tail preservation，并覆盖 TeaCache extractor 与 `rope_table=None` 分支。当前 PR 新增的仅是独立 fused-kernel BF16 测试，没有 H3 集成或 TeaCache 数值回归，因此不能据此宣称模型端到端 parity。^[PR #5990] + 在 NPU，BF16 packed table 必须 request-/SP-rank-local，且 shape、width、device、dtype 与本 rank span 一致；不能重建 model-global table 或把复用写成通用性能保证。^[PR #6410] ## MMH3-1i — Qwen3-VL 文本编码器 NPU RoPE 必须保持 BNSD 与逐样本 M-RoPE @@ -185,6 +186,7 @@ confidence: high - 触发：修改 MiniMax-H3 的 strict Ulysses、`_embed()`、packed RoPE、`local_sp_prepare`、DiT final projection 或 `sp_gather` 边界。 - 强制：只有共享 strict-Ulysses gate 通过时，按 rank 的 packed span 先筛选 image/audio latent rows；token refiner 仍处理完整 text context 后再筛选 text rows；构造局部 embedding 与对应局部 RoPE rows，使用 `local_sp_prepare` 运行 blocks。局部 transformer 输出必须先执行 final projection，再拼接 video/audio logits 并 gather compact FP32 rows；恢复后按 architecture output width 拆分并执行原有 target/condition row selection。其余路径保留完整 embedding、RoPE、`sp_prepare`、hidden gather 后 projection 的旧顺序。 +- 强制：TeaCache 的 strict-SP 路径保持 modulation input、hidden state、residual 与 cache decision 为 rank-local；以 SP MAX reduction 同步是否 compute，cache-forced rank 必须重置累计 L1 state，compact projected logits 只 gather 一次。非 strict/global 路径仍保留一次 hidden-state gather。^[PR #6410] - 禁止：在局部路径 gather `[S, 5376]` BF16 hidden state；绕过 `sp_input---local_sp_prepare` hook；只截取 text embedding 而不先运行完整 refiner；修改 Ulysses Q/K/V all-to-all、attention backend、packed-prefix 或 scheduler 语义；把约 21 倍 gather payload 降低写成端到端性能保证。 - 验收：模型合同测试须证明各 rank 的局部 multimodal rows 拼接等于完整 embedding、RoPE 行数与 span 对齐、所有 fallback 恢复旧路径，并断言 projection-before-gather 的 payload width 与最终 logits/target/condition rows 等价；性能和 video/audio quality 只能在固定 PR workload、硬件、steps、seed 与 gate 下复核，不能外推到其他 task 或拓扑。 ^[PR #6173] From 2648cc1a6e2e5f6641f3183cbd1182db73134440 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:28:24 +0800 Subject: [PATCH 374/465] knowledge(vllm-omni): sync 5a5bc55c (PR #6573) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-checkpoint-loading.md | 10 +++++----- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index aeb44ef5c..d62ac3494 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: bb5bf8fd0af4dcd84496e16b24cb5ce6f4d7fb17 + previous_audited_sha: d9980c75e8fe68ba2e6aa84360f109182a75023b audited_ref: main - audited_sha: d9980c75e8fe68ba2e6aa84360f109182a75023b + audited_sha: 5a5bc55c25b9626c8cfeef0f913de285e888a6e1 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7a59e4b61..637ea0a97 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d9980c75e8fe68ba2e6aa84360f109182a75023b`. The canonical machine baseline +5a5bc55c25b9626c8cfeef0f913de285e888a6e1`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index dd88b8e78..0e28a9e6b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d9980c75` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5a5bc55c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index fbe59fac2..5af178178 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d9980c75` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 5a5bc55c` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md index b07dab720..788b1e140 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-checkpoint-loading.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6162", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", "PR #6651", vllm_omni/diffusion/model_loader/, vllm_omni/quantization/, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, vllm_omni/quantization/svdquant_config.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/offloader/, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #6722", "PR #4061"] +sources: ["PR #5087", "PR #5088", "PR #5136", "PR #5544", "PR #5677", "PR #5737", "PR #5764", "PR #5802", "PR #5836", "PR #5839", "PR #5848", "PR #5872", "PR #5910", "PR #6070", "PR #6162", "PR #6234", "PR #6279", "PR #6445", "PR #6486", "PR #6591", "PR #6651", "PR #6573", vllm_omni/diffusion/model_loader/, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/quantization/, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, vllm_omni/quantization/svdquant_config.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/offloader/, tests/diffusion/model_loader/test_diffusers_loader.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, "PR #5531", "PR #6722", "PR #4061"] confidence: high --- @@ -147,12 +147,12 @@ confidence: high 相关执行流见 [Diffusion architecture](architecture.md)；component lifecycle 见 [component lifecycle 规则](rules-component-lifecycle.md)。 -## DIFF-2t — DLO AllGather 只允许已验证的 per-tensor online FP8 布局 +## DIFF-2t — DLO AllGather 只允许已验证的 online quant 布局 -- 触发：修改 DLO+AllGather 与在线量化的兼容门禁、普通 loader fallback，或 FP8 权重/scale 的分片重建。 -- 强制：仅允许 `Fp8PerTensorOnlineLinearMethod` 进入 DLO+AllGather；由于 direct checkpoint mmap 不能生成运行时量化布局，必须先由普通 loader 完成 FP8 权重和 scale，再交给 DLO 按 dtype 分片，并按记录的 shape、stride 重建转置 Cutlass 权重。每个 rank 启动时可能暂存完整 FP8 模型，运行时才保留 DLO shard。 +- 触发：修改 DLO+AllGather 与在线量化的兼容门禁、普通 loader fallback，或 FP8/INT8/MXFP8 权重与 scale 的分片重建。 +- 强制：真实 DLO weight collective（effective group size > 1）只允许 `Fp8PerTensorOnlineLinearMethod`、`Int8OnlineLinearMethod`、`NPUInt8OnlineLinearMethod`，以及模块可导入时的 `NPUMxfp8OnlineLinearMethod`/`VllmMxfp8OnlineLinearMethod`；必须先由普通 loader 完成 runtime weight/scale finalization，再按物理 storage 顺序分 dtype 打包，并按记录的 shape、stride 重建。effective group size=1 不执行 collective，因此不应用该 allowlist；direct checkpoint mmap/HWR 不生成这些 runtime layout，其他 online method 在真实 AllGather 下全部 fail closed。 - 禁止：把该 allowlist 扩展到未经验证的在线量化方法；绕过普通 loader 直接 mmap 在线量化权重；丢失非连续权重的物理 stride；将 transient 完整模型 host memory 峰值描述为运行时常驻开销。此前 DIFF-2s 对所有 runtime-created FP8 的 AllGather 禁止由本规则收窄覆盖。 -- 验收：loader 测试必须证明 allowlist 分支确实执行、普通 loader 在 DLO 前完成权重与 scale 处理，并对未验证 method fail fast；DLO 测试必须验证 FP8 weight/scale 的 dtype、值、shape 和转置 stride 重建。另覆盖 no-AllGather 路径及至少一次真实多 rank AllGather smoke；声明质量或性能 parity 仍需固定模型、硬件和 exact workload 的独立证据。^[PR #6279] +- 验收：loader 测试必须证明 allowlist 分支确实执行、普通 loader 在 DLO 前完成权重与 scale 处理、group=1 跳过门禁，并对未验证 method fail fast；DLO 测试逐类验证 weight/scale 的 dtype、值、shape 与 contiguous/转置 stride 重建。PR #6573 只有 mock collective/unit layout 覆盖，没有真实 multi-rank online-quant DLO E2E；补齐该 smoke 以及固定模型、硬件、checkpoint、workload 的输出质量/性能对照前，不得升级为端到端 parity 声明。PR 中 B300 H3 默认测试也未启用 DLO/online quant，不能充当该组合证据。^[PR #6279] ^[PR #6573] ## DIFF-2u — diffusion 并行状态必须保持单一拓扑所有权并原子清理 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 514042b05..5021c15e2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d9980c75` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5a5bc55c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0ab71d59c..c60e0c9f6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d9980c75` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5a5bc55c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 12c72ff53..a163894e4 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d9980c75` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5a5bc55c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f0c46d4eb..cd45db96d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d9980c75` 验证存在 +- 源码校验：以上路径均已在 `main @ 5a5bc55c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d127ead76..ac8dfbff6 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d9980c75`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5a5bc55c`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from bb5bf8fd \ - --to d9980c75 \ + --from d9980c75 \ + --to 5a5bc55c \ --repo \ --mode report-only ``` From 63bc2a8cfe05b66e4eee8235be2969d313b264eb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:31:13 +0800 Subject: [PATCH 375/465] knowledge(vllm-omni): sync a5b8c827 (PR #6757) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md | 5 ++++- 10 files changed, 16 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d62ac3494..e2ef6646a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d9980c75e8fe68ba2e6aa84360f109182a75023b + previous_audited_sha: 5a5bc55c25b9626c8cfeef0f913de285e888a6e1 audited_ref: main - audited_sha: 5a5bc55c25b9626c8cfeef0f913de285e888a6e1 + audited_sha: a5b8c827887cf75e039f09452f4ac885c26f0234 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 637ea0a97..51b45da89 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5a5bc55c25b9626c8cfeef0f913de285e888a6e1`. The canonical machine baseline +a5b8c827887cf75e039f09452f4ac885c26f0234`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0e28a9e6b..8176fffce 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5a5bc55c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a5b8c827` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5af178178..f3c15ee66 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5a5bc55c` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ a5b8c827` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5021c15e2..9bbb55e49 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5a5bc55c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a5b8c827` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c60e0c9f6..3482909b1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5a5bc55c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a5b8c827` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a163894e4..577250f98 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5a5bc55c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a5b8c827` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index cd45db96d..1fd3aaed7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5a5bc55c` 验证存在 +- 源码校验：以上路径均已在 `main @ a5b8c827` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ac8dfbff6..98d3e8d55 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5a5bc55c`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a5b8c827`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from d9980c75 \ - --to 5a5bc55c \ + --from 5a5bc55c \ + --to a5b8c827 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 65d1073f1..6f2b81cec 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757"] confidence: high --- @@ -280,6 +280,9 @@ confidence: high 并记录实际执行结果及硬件；local checkpoint 与 cache fallback 各一例。changed tests、marker 或 parametrization 只能证明 collection 意图；没有实际 runtime 结果时，不能声称 CUDA/NPU、 音频质量或 flake 已闭环。^[PR #6056] + `duplex_camera_frames()` 必须解包 `_video_frames_from_file()` 的 `(base_frames, stacked_frames)`，只返回 + flat base-frame list；这是 CPU fixture input-shape contract，不改变 runtime camera stacking、H100 E2E + 或模型行为。^[PR #6757] 共享 bridge/batch 规则见 [Model Executor rules](../../components/model-executor/rules.md)； 公开入口完整性见 [model adaptation guardrails](../../review/guides/model-adaptation-guardrails.md)。 From 836185bf8ce4f15e62e8f102de04d666d41cda13 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:34:51 +0800 Subject: [PATCH 376/465] knowledge(vllm-omni): sync c50b7de1 (PR #6037) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/diffusion/rules-attention.md | 11 +++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/minimax-h3/rules-rainfusion.md | 9 ++++++++- 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e2ef6646a..cf0816245 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5a5bc55c25b9626c8cfeef0f913de285e888a6e1 + previous_audited_sha: a5b8c827887cf75e039f09452f4ac885c26f0234 audited_ref: main - audited_sha: a5b8c827887cf75e039f09452f4ac885c26f0234 + audited_sha: c50b7de16712fe02a1a7c9e1b69458553ba1c9a7 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 51b45da89..f11ebb679 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -a5b8c827887cf75e039f09452f4ac885c26f0234`. The canonical machine baseline +c50b7de16712fe02a1a7c9e1b69458553ba1c9a7`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8176fffce..19e4ae144 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a5b8c827` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c50b7de1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f3c15ee66..666dac55f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a5b8c827` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ c50b7de1` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 9f2207e15..b6f449f84 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6518", docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] +sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] confidence: high --- @@ -92,6 +92,10 @@ confidence: high 32×128 rows 均保持 FlashAttention dense fallback；显式错 layout、causal 或 Ring>1 仍 fail closed。非 8×8 空间网格的 residual 被 always-kept，因而 realized sparsity 低于 nominal；潜空间 h/w 为 8 的倍数（输入高宽为 256 的倍数）仍是性能首选。 +- 边界：`end_step>0` 时，forward context 必须同时提供当前 `step_idx` 与 `total_denoise_steps`， + 并在 `step_idx >= total_denoise_steps-end_step` 的最后窗口回到 dense；默认 `end_step=0`。 + `precision` 只允许 `bf16|fp8|mix` 且默认 `bf16`；non-BF16 请求必须确认已安装 MindIE-SD 的 + `sparse_attention` 显式声明 `precision` 参数，否则 fail fast，不能让 `**kwargs` 静默吞掉配置。 - 禁止：只因 `mindiesd` 可 import 就断言 irregular-tail 或 multi-span 合同可用；availability 必须在 模型构造前检查 `sparse_attention` 明确接受 `video_spans`，否则以可操作的 upgrade/FLASH_ATTN error fail closed，不能成功启动后在 T2VA/FL2VA/Ref2VA dispatch 才 TypeError，也不能静默尝试 legacy @@ -102,7 +106,10 @@ confidence: high realized sparsity 或加速。CPU tests 还应覆盖 multi-span 的 used length、geometry/role/overlap/bounds、 total threshold 与 clip-boundary dense-context fallback；它们只证明 plan payload。PR #6518 review 的 MindIE-SD `2661f83` field-name/layout check 与 4×B300 FLASH_ATTN regression 都不执行 RainFusion； - 真实 Ref2VA same-seed sparse quality 仍未覆盖。^[PR #6000] ^[PR #6518] + 真实 Ref2VA same-seed sparse quality 仍未覆盖。另覆盖 `end_step` 边界、zero-tail、precision enum + normalization 与旧 MindIE-SD fail-fast；serial/homogeneous producer 必须发布并清理 progress trio。 + heterogeneous batch 的空 progress trio 在该目标没有独立 dense guard，不能据此声称安全 fallback。 + ^[PR #6000] ^[PR #6518] ^[PR #6037] ## DIFF-1k — 非等长 SP attention 必须保留 K/V mask 并走 mask-safe backend diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9bbb55e49..362485897 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a5b8c827` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c50b7de1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 3482909b1..378ef66e7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a5b8c827` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c50b7de1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 577250f98..89a2671a9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a5b8c827` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c50b7de1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1fd3aaed7..08a1041bb 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a5b8c827` 验证存在 +- 源码校验：以上路径均已在 `main @ c50b7de1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 98d3e8d55..77348d3ff 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a5b8c827`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c50b7de1`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 5a5bc55c \ - --to a5b8c827 \ + --from a5b8c827 \ + --to c50b7de1 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-rainfusion.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-rainfusion.md index 6f85a48bc..7478eeab6 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-rainfusion.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-rainfusion.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5706", "PR #6000", "PR #6518", vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py] +sources: ["PR #5706", "PR #6000", "PR #6037", "PR #6518", vllm_omni/diffusion/attention/backends/abstract.py, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py] confidence: high --- @@ -31,6 +31,13 @@ confidence: high - 禁止：把 nominal sparsity 当 realized sparsity 或质量保证；未声明 layout 时不能让 sparse path 假设 BSND 而 dense fallback 解释成 BNSD。INT8、RainFusion、no-AllGather DLO 可组合，但 online quantization 不得与 DLO+AllGather 组合；本 PR 没有证明 HSDP 或其他 quantizer 的组合语义。 +- 强制：`end_step` 使 denoise 形成 dense→sparse→dense：当 `end_step>0` 且 + `step_idx >= total_steps-end_step` 时保持 dense；`end_step=0` 不启用 tail fallback。serial 与处于 + 同一 progress point 的 homogeneous batch 必须发布 `(step, sigma, total_steps)`，请求结束后清空三者。 + heterogeneous batch 发布 `(None,None,None)`；目标实现没有单独用缺失 progress 阻断 sparse plan， + 因而不能把该状态宣称为已证明的 dense fallback。`precision` 只接受 `bf16|fp8|mix`，默认 + `bf16`；non-BF16 只有在 MindIE-SD `sparse_attention` 的显式 signature 含 `precision` 时才可启用， + 不能信任会吞掉未知 kwargs 的旧实现。^[PR #6037] - 验收：CPU plan tests 覆盖 aligned/irregular 均进 sparse plan、tail closure、min length、layout 和 skip/step；NPU 条件数值测试以 `sparsity=0` 直接调用 kernel 对 dense reference，mean relative error 阈值为 `2e-3`；它不证明 `sparsity=0.8` 质量 parity。PR #6518 的 CPU tests 覆盖 Ref2VA multi-span From f015346d615d86ea65f567630a3dc96255d6e196 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:38:02 +0800 Subject: [PATCH 377/465] knowledge(vllm-omni): sync e338f0ec (PR #6650) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 4 +++- knowledge/repos/vllm-omni/ci/guides/test-tiers.md | 9 +++++---- knowledge/repos/vllm-omni/ci/rules.md | 4 +++- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 12 files changed, 23 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cf0816245..150663b70 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a5b8c827887cf75e039f09452f4ac885c26f0234 + previous_audited_sha: c50b7de16712fe02a1a7c9e1b69458553ba1c9a7 audited_ref: main - audited_sha: c50b7de16712fe02a1a7c9e1b69458553ba1c9a7 + audited_sha: e338f0ece35b536173ec3365745eb95f2303528f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f11ebb679..d8fd9d545 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c50b7de16712fe02a1a7c9e1b69458553ba1c9a7`. The canonical machine baseline +e338f0ece35b536173ec3365745eb95f2303528f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 702828e46..616fe49c6 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464", "PR #6730", "PR #6745", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742"] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464", "PR #6730", "PR #6745", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742", "PR #6650", pyproject.toml, tests/helpers/mark.py, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py] --- # vLLM-Omni CI @@ -12,6 +12,8 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie ## 什么时候查这里 - 处理 vLLM-Omni 的 L2/L4、模型测试配置或 CI 特有问题。 +- 处理 `@hardware_test`/`hardware_marks`、SKU 与 `cards_N` 自动 marker 或按卡数切分时，先看 + [测试分级与 markers](guides/test-tiers.md)，再按 OMNI-CI-1a 审核 lane 的真实收集与执行。 ## MiniCPM-o 4.5 full-payload regression coverage diff --git a/knowledge/repos/vllm-omni/ci/guides/test-tiers.md b/knowledge/repos/vllm-omni/ci/guides/test-tiers.md index 62b81cc48..782827fa5 100644 --- a/knowledge/repos/vllm-omni/ci/guides/test-tiers.md +++ b/knowledge/repos/vllm-omni/ci/guides/test-tiers.md @@ -1,10 +1,10 @@ --- title: "测试分级（L1–L5）与 pytest markers" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-05 type: guide tags: [vllm-omni, ci] -sources: [docs/contributing/ci/test_system_overview.md, docs/contributing/ci/test_writing_guide.md] +sources: [docs/contributing/ci/test_system_overview.md, docs/contributing/ci/test_writing_guide.md, "PR #6650"] --- # 测试分级（L1–L5）与 pytest markers @@ -31,8 +31,9 @@ sources: [docs/contributing/ci/test_system_overview.md, docs/contributing/ci/tes - 分级：`core_model`（L1&L2）、`advanced_model`（L3）、`full_model`（L4）。 - 领域：`diffusion`、`omni`、`tts`、`cache`、`parallel`。 - 平台：`cpu`、`gpu`、`cuda`、`rocm`、`xpu`、`npu`。 -- 硬件：`H100`、`L4`、`MI325`、`A2`、`A3`；多卡：`distributed_{cuda,rocm,npu}`； - 条件跳过：`skipif_{cuda,rocm,npu}`（卡数不足时跳过）。 +- 硬件：SKU 与 `cards_N` card-count markers 均由 `pyproject.toml` taxonomy/`hardware_test` 或 + `hardware_marks` 派生（default `cards_1`）；Buildkite 按 `cards_N` 分区。handwritten SKU marks + 由 `check-mark` pre-commit 拒绝；这是 collection/routing contract，不是 runtime claim。^[PR #6650] - 其他：`slow`（快速 CI 可跳）、`benchmark`。 - 带 `*` 的平台/硬件/分布式/skipif markers 由 `@hardware_test` 参数化装饰器（或 `hardware_marks`）**自动添加**，用法： diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 713602bbf..e97e06a22 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-05 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", .buildkite/common/scripts/run_cov_split.sh, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743"] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743"] confidence: high --- @@ -29,6 +29,8 @@ confidence: high ## OMNI-CI-1a — 硬件 lane 必须真实收集并执行目标路径 - 触发：增加模型、硬件路径、nightly/ready/merge lane，或升级运行时/镜像。 +- 强制：硬件 helper 从 `pyproject.toml` 派生 platform/SKU/`cards_N`，default 为 `cards_1`；Buildkite + selector 按 cards count 切分，直接写 SKU marker 被 pre-commit 拒绝。^[PR #6650] - 强制：资源数、pytest marker、pipeline 路由和共享 stage registry 同步；必需依赖、模型初始化 或 collection 失败必须让 job 失败，并证明目标测试数大于零且原失败在对应硬件消失。默认 `prepend` import mode 下，whole-tree collection 遇到同名叶子目录时，所有中间父目录必须用 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 19e4ae144..4c06b02ed 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c50b7de1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e338f0ec` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 666dac55f..25340e1a5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c50b7de1` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ e338f0ec` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 362485897..26a5056e1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c50b7de1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e338f0ec` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 378ef66e7..74def338d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c50b7de1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e338f0ec` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 89a2671a9..42ae76193 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c50b7de1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e338f0ec` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 08a1041bb..36d0cdae2 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c50b7de1` 验证存在 +- 源码校验：以上路径均已在 `main @ e338f0ec` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 77348d3ff..5360d2391 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c50b7de1`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e338f0ec`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from a5b8c827 \ - --to c50b7de1 \ + --from c50b7de1 \ + --to e338f0ec \ --repo \ --mode report-only ``` From d4a1b8ef431bdf5a1e836102c93910531c371800 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:41:22 +0800 Subject: [PATCH 378/465] knowledge(vllm-omni): sync 72a1ce48 (PR #6696) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../benchmark/guides/omniinteract-realtime.md | 12 ++++++++++-- knowledge/repos/vllm-omni/ci/rules.md | 5 ++++- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 26 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 150663b70..9a654c6a7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c50b7de16712fe02a1a7c9e1b69458553ba1c9a7 + previous_audited_sha: e338f0ece35b536173ec3365745eb95f2303528f audited_ref: main - audited_sha: e338f0ece35b536173ec3365745eb95f2303528f + audited_sha: 72a1ce488bf72edae6315a9d63d54b36018d8ccb # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d8fd9d545..3f3ca1135 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e338f0ece35b536173ec3365745eb95f2303528f`. The canonical machine baseline +72a1ce488bf72edae6315a9d63d54b36018d8ccb`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md b/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md index 8ee9b7bb2..c3540b36b 100644 --- a/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md +++ b/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md @@ -1,10 +1,10 @@ --- title: "OmniInteract Realtime Benchmark" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: guide tags: [vllm-omni, benchmark] -sources: ["PR #6522", docs/cli/bench/serve.md, vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/entrypoints/cli/benchmark/cli_args.py] +sources: ["PR #6522", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/experimental/fullduplex/openai/protocol.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, docs/cli/bench/serve.md, vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/entrypoints/cli/benchmark/cli_args.py] confidence: high --- @@ -51,6 +51,11 @@ production deployment contract. ^[PR #6522] timing; ITL is reported only when all required intervals exist. Missing measurements remain missing rather than becoming zero, so request goodput cannot pair metrics from different responses. +- TPOT 样本必须 finite 且严格大于零：优先使用长度完整的 exact ITL；否则只有各 Stage-0 + `output_token_count` 合计与最终 token 数完全一致、且每段 `tpot_ms` 都是正有限值时，才按 decode + interval 数加权恢复 TPOT。native duplex 的逐段 metrics 是增量，需累加；chat fallback 发出的是 + cumulative snapshot，需按 stage 替换最新值。上述来源都不可用时，仅当 client 观察到 finite + `text_latency > ttft > 0` 才可按总 token 数回退计算，否则 TPOT 保持 unavailable。^[PR #6696] ## Artifact and evidence contract @@ -68,3 +73,6 @@ production deployment contract. ^[PR #6522] commit adds no performance threshold consumer, no answer-quality evaluation, and no 12-video nightly/production configuration. Do not promote its example command, default model, or smoke result to a runtime guarantee. +- 若某 hardware baseline 配置了 `mean_tpot_ms`，结果必须同时有正的 `num_tpot_samples` 和 finite + aggregate；仅有字段或 `NaN` 不能通过 gate。目标 JSON serializer 仍允许 non-standard `NaN`， + 因而 artifact consumer 仍需显式拒绝，不能把序列化成功当作 strict JSON/metric validity。^[PR #6696] diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index e97e06a22..ddcbacdcc 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-05 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743"] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py] confidence: high --- @@ -212,6 +212,9 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload ## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 - 触发：修改 perf JSON 的 `baseline`、硬件 label/marker、benchmark result schema，或恢复性能阈值断言。 +- 强制：TPOT 只接受 finite positive value；优先 exact ITL，其次 Stage0 weighted fallback，最后仅在 + `text_latency > ttft > 0` 时用 latency fallback。native duplex 累加 segment counts，chat cumulative + snapshots 替换；configured mean TPOT baseline 需 positive sample count 和 finite aggregate。JSON 仍非 strict，NaN gap 未闭合。^[PR #6696] - 强制：baseline 只接受 `{hardware: {metric: threshold}}` 的硬件嵌套结构；顶层 hardware 必须是 `pyproject.toml` 中带 `[hardware-resource]` 标签的 pytest marker，每个 bucket 非空。metric 名可 自定义，值只能是单点 scalar 或与 `request_rate` / `max_concurrency` 顺序严格对齐的 list。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4c06b02ed..301f3158f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e338f0ec` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 72a1ce48` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 25340e1a5..ddf250479 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e338f0ec` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 72a1ce48` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 26a5056e1..cfafcd315 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e338f0ec` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 72a1ce48` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 74def338d..736aa3f57 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e338f0ec` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 72a1ce48` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 42ae76193..bbf10636f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e338f0ec` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 72a1ce48` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 36d0cdae2..bbd8edba1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e338f0ec` 验证存在 +- 源码校验：以上路径均已在 `main @ 72a1ce48` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5360d2391..8f797e452 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e338f0ec`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 72a1ce48`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from c50b7de1 \ - --to e338f0ec \ + --from e338f0ec \ + --to 72a1ce48 \ --repo \ --mode report-only ``` From 961bdfc274a58e647ea65de625ebd4ec3329fdfb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:44:43 +0800 Subject: [PATCH 379/465] knowledge(vllm-omni): sync 676a35fa (PR #6767) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/minicpm-o-4-5/rules-duplex.md | 11 ++++++++--- 10 files changed, 20 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9a654c6a7..4d353f1f9 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e338f0ece35b536173ec3365745eb95f2303528f + previous_audited_sha: 72a1ce488bf72edae6315a9d63d54b36018d8ccb audited_ref: main - audited_sha: 72a1ce488bf72edae6315a9d63d54b36018d8ccb + audited_sha: 676a35fa8bee9a261a9ae12e7ebab46e80ce0f0e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3f3ca1135..d4f75a45c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -72a1ce488bf72edae6315a9d63d54b36018d8ccb`. The canonical machine baseline +676a35fa8bee9a261a9ae12e7ebab46e80ce0f0e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 301f3158f..428f601aa 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 72a1ce48` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 676a35fa` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ddf250479..d2ec7e735 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 72a1ce48` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 676a35fa` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index cfafcd315..f9df4bf76 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 72a1ce48` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 676a35fa` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 736aa3f57..59d66b85c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 72a1ce48` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 676a35fa` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index bbf10636f..028e26677 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 72a1ce48` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 676a35fa` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index bbd8edba1..441a6a621 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 72a1ce48` 验证存在 +- 源码校验：以上路径均已在 `main @ 676a35fa` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8f797e452..6fac2b740 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 72a1ce48`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 676a35fa`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from e338f0ec \ - --to 72a1ce48 \ + --from 72a1ce48 \ + --to 676a35fa \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md index c30c66c9c..4e638157b 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] +sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", "PR #6767", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/deploy/minicpmo_4_5_2gpu.yaml, vllm_omni/deploy/minicpmo_4_5_3gpu.yaml, vllm_omni/deploy/minicpmo_4_5_8x4090.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] confidence: high --- @@ -47,9 +47,14 @@ confidence: high ## MCPMO-4e — shipping profile 必须共服 chat 与 native duplex - 触发：修改任一 `minicpmo_4_5*.yaml`、duplex session placement 或 Stage 1 Talker 默认/运行时采样参数。 -- 强制：所有 shipping profile 设置 `session_mode=duplex`、`active_stream_window=1`，且 `duplex_session.max_sessions` 与 Stage 0/1 的 `max_num_seqs` 容量一致；replica overlay 从 base 继承这些字段。YAML 保留 chat 的 Stage 1 `min_tokens=50` 与既有 codec 参数，native duplex adapter 仅在该 session 的 runtime config 覆盖 `min_tokens=0`。AR stage 保留默认 async scheduler。 +- 强制：所有 shipping profile 设置 `session_mode=duplex`，且 `active_stream_window` 只限制 + first-packet admission，不是 session lifetime cap。default/2gpu/3gpu profile 的 window、 + `duplex_session.max_sessions` 与 Stage 0/1 `max_num_seqs` 均为 4；8×4090 profile 因 24GB + 资源边界保持 window/stage capacity 1，并沿用 default session capacity 1。replica overlay 从 base + 继承这些字段。YAML 保留 chat 的 Stage 1 `min_tokens=50` 与既有 codec 参数，native duplex adapter + 仅在该 session 的 runtime config 覆盖 `min_tokens=0`；AR stage 保留默认 async scheduler。^[PR #6767] - 禁止：恢复独立 `minicpmo_4_5_duplex.yaml`；在 replica overlay 重复并漂移 session 配置；为 duplex 全局关闭 async scheduling；或把 runtime-only floor 写回 YAML 而改变 chat TTS。 -- 验收：展开四份 base profile 及三个 replica overlay，断言全部 duplex-enabled、窗口与 session capacity 正确；同一 server 分别验证 `/v1/realtime?duplex=1` 和 chat，前者 Stage 1 看到 `min_tokens=0`、后者仍看到 `50`。删除旧 overlay 是部署文件名迁移，外部显式引用必须同步更新。^[PR #6458] ^[PR #6619] +- 验收：展开四份 base profile 及三个 replica overlay，断言全部 duplex-enabled、窗口与 session capacity 正确；同一 server 分别验证 `/v1/realtime?duplex=1` 和 chat，前者 Stage 1 看到 `min_tokens=0`、后者仍看到 `50`。删除旧 overlay 是部署文件名迁移，外部显式引用必须同步更新。PR #6767 的 L20X concurrency=4 观察只证明该精确 profile/workload 的 first-packet admission，不构成通用吞吐、延迟或硬件保证。^[PR #6458] ^[PR #6619] ^[PR #6767] - admission/expiry E2E probe 必须从实际启动的 deploy YAML 经 `load_deploy_config(...).duplex_session.max_sessions` 取得 admission limit，不能保留 From 9243758c327df77a1492b3b399719b8613875d4f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:47:21 +0800 Subject: [PATCH 380/465] knowledge(vllm-omni): sync 3adee400 (PR #6728) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../repos/vllm-omni/models/qwen3-tts/rules.md | 22 ++++++++++++++++++- 10 files changed, 33 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4d353f1f9..bd509e7d4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 72a1ce488bf72edae6315a9d63d54b36018d8ccb + previous_audited_sha: 676a35fa8bee9a261a9ae12e7ebab46e80ce0f0e audited_ref: main - audited_sha: 676a35fa8bee9a261a9ae12e7ebab46e80ce0f0e + audited_sha: 3adee400a03454d483a7c07018f4c65a5b8ab8f9 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d4f75a45c..e056e3c1f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -676a35fa8bee9a261a9ae12e7ebab46e80ce0f0e`. The canonical machine baseline +3adee400a03454d483a7c07018f4c65a5b8ab8f9`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 428f601aa..6ca23ffa3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 676a35fa` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3adee400` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d2ec7e735..75fd46d57 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 676a35fa` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 3adee400` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f9df4bf76..94869e36d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 676a35fa` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3adee400` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 59d66b85c..c1b7550a3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 676a35fa` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3adee400` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 028e26677..13a0737bf 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 676a35fa` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3adee400` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 441a6a621..ffb681a48 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 676a35fa` 验证存在 +- 源码校验：以上路径均已在 `main @ 3adee400` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6fac2b740..b590a8d7b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 676a35fa`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3adee400`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 72a1ce48 \ - --to 676a35fa \ + --from 676a35fa \ + --to 3adee400 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index 9c173b66e..51168dd78 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6523", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] +sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6523", "PR #6728", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_speech_stream.py, vllm_omni/entrypoints/openai/speech_usage.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_serving_speech_stream.py, tests/entrypoints/openai_api/test_tts_adapter.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] confidence: high --- @@ -178,3 +178,23 @@ Qwen 家族入口见 [Qwen-Omni](../qwen-omni/_index.md)。 - 强制：默认 `silence_ban_frames=0`；加载 checkpoint 后通过 `_encode_ref_audio_batch` 编码多种静音样本并只收集 codebook-0 ids，词表为空、越界或明显过大时记录 warning、清空启用值并禁用功能。decode 时只对 Base 的 x-vector-only request 在前 N 个 history steps 屏蔽派生 ids；mode resolution 必须与 prompt builder 一致，`voice_clone_prompt.icl_mode` 覆盖和未记录 mode 都要正确处理。 - 禁止：硬编码当前 checkpoint 的 12 个 token、把 mask 施加到 ICL 或非 Base 请求、对不可信派生词表只做部分 masking，或把经验值 `N=3` 当成跨 checkpoint 的固定最优参数。 - 验收：覆盖默认关闭、有效派生、空/越界/ oversized 派生和 encoder failure；覆盖 step 边界、mixed x-vector/ICL batch、ICL 与非目标请求保持未修改，以及实际 stage 配置能到达 talker。^[PR #5048] + +## Q3TTS-4b — Base codec token exhaustion 必须丢弃不完整音频 + +- 触发：修改 Qwen3-TTS Base 的 codec token budget、Stage-0 finish metadata、speech generation + validation、非流式 retry 或 streaming terminal/error 语义。 +- 强制：caller 未显式给 `max_new_tokens`、task 为 Base 且 text token count>0 时，自动上限为 + `min(configured_cap, max(192, 12*text_tokens))`（无 configured cap 时取 dynamic cap）；tokenizer + 不可用或计数失败时保留 configured cap，非 Base 也不套 text-scaled ceiling。无论上限来自 caller、 + config 或动态计算，只在 request metadata 记录了 effective cap、task 为 Base 且 Stage 0 实际 + `finish_reason="length"` 时判为 budget exhaustion；不能从 token 数接近上限自行猜测。 +- 强制：exhausted generation 的音频不完整，non-streaming 只能在 caller 同时未指定 seed 与 + `max_new_tokens` 时，用新 request id 和 fresh seed 重试恰好一次；不得修改原 request。raw audio、 + SSE 与 WebSocket 等 streaming 路径不重试：若已经发出 delta，terminal error 必须标记 + `partial_audio=true, action=discard`，且不得发 success/done；错误发生在首个 audio 前则不伪造 partial。 +- 禁止：把长度终止的 Base 音频当成功结果；用 fresh seed 覆盖显式可复现请求；在 streaming 中 + 隐式重放造成重复音频；把该 guard 外推到所有 TTS adapter 或声称 dynamic cap 保证 EOS。 +- 验收：覆盖 short/long text、configured cap、显式 max、token-count failure 与非 Base；分别验证 + EOS/非 length 正常通过、length+recorded-cap 拒绝、非流式一次 retry 及显式 seed/max 不 retry； + raw/SSE/WebSocket 均携带 terminal metrics，并覆盖 partial/no-partial error。目标回归证明控制流和 + 丢弃语义，不证明任意 prompt 必达 EOS、音频质量或最佳 cap 比例。^[PR #6728] From 33d7d7c723aab98e2fadb765e11f4eb1601d9c5b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:51:15 +0800 Subject: [PATCH 381/465] knowledge(vllm-omni): sync 87736a3c (PR #6626) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../vllm-omni/components/distributed/rules.md | 16 ++++++++++++- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 17 ++++++++++++-- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../models/minicpm-o-4-5/rules-duplex.md | 23 ++++++++++++++++++- 12 files changed, 64 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bd509e7d4..e21ad234a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 676a35fa8bee9a261a9ae12e7ebab46e80ce0f0e + previous_audited_sha: 3adee400a03454d483a7c07018f4c65a5b8ab8f9 audited_ref: main - audited_sha: 3adee400a03454d483a7c07018f4c65a5b8ab8f9 + audited_sha: 87736a3cc50a96b5c6e83e631c7b4d3a256fcae0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e056e3c1f..4b2a6a90f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3adee400a03454d483a7c07018f4c65a5b8ab8f9`. The canonical machine baseline +87736a3cc50a96b5c6e83e631c7b4d3a256fcae0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6ca23ffa3..68cfe5cb3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3adee400` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 87736a3c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 75fd46d57..8e64c81f6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3adee400` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 87736a3c` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 94869e36d..b8f887360 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3adee400` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 87736a3c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 1b867cfa4..87164a783 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-05 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406"] +sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626"] confidence: high --- @@ -90,6 +90,20 @@ confidence: high - 禁止：`cleanup_sender` 清空 map 后让 pending save 用默认 chunk counter 写回旧 chunk 0；让 scheduler thread 等待 connector I/O；把同一 external ID 的新 generation 误当作旧 task 的 cleanup target。 - 验收：覆盖 pending/in-flight abort、stale put completion、ID reuse 和普通 async-chunk transport。^[PR #6089] +## DIST-1i — receiver registration 必须隔离旧 poll 与新 request generation + +- 触发：async-chunk receiver 注册/清理与 `connector.get()` 并发、同一 internal request ID 恢复或复用， + 或已消费 payload 需在 scheduler thread 才能安全修改 prompt。 +- 强制：每次 receiver registration 持有独立 identity entry；`connector.get()` 可在锁外阻塞，但结果 + commit 必须在 receiver-state lock 下确认当前 map 仍指向同一 entry。cleanup 是 commit barrier：旧 + queued/in-flight poll 的迟到结果只能丢弃，不能重建已删除或新 generation 的 prompt/window state。 + AR payload 先冻结为 pending update，再由 scheduler thread 用当前 live request 应用；contract error + 进入一次性 receive-failure ledger，不得重取已消费 chunk。 +- 禁止：只按可复用 request-id 判断 poll 仍有效；在 I/O 期间持有 scheduler-facing lock；让旧 + terminal cleanup 清掉新 registration；使用被移出 scheduler 的 fallback request 应用 pending mutation。 +- 验收：覆盖 cleanup-during-get、same-ID re-registration、late success/error、重复 register/cleanup、 + scheduler live-ID replacement 与 invalid payload ledger drain；证明旧 entry 不影响新 request。^[PR #6626] + ## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 - 触发：上游 MoE factory/oracle 或 `init_model_parallel_group` 增加 all-to-all manager 要求。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c1b7550a3..7ba219d19 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3adee400` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 87736a3c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 13a0737bf..d05b1beca 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3adee400` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 87736a3c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 321f23c8b..9fafe2ab2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -1,10 +1,10 @@ --- title: "Scheduler 规则" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619", "PR #6680"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/core/sched/test_omni_scheduler_mixin_timeouts.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619", "PR #6680", "PR #6626"] --- # Scheduler 规则 @@ -27,6 +27,7 @@ PR 描述先命中下表，再打开对应规则组和首批源码；changed fil | sampled-token logprobs、spec decode trim、request-local output error | SCHED-5a | `core/sched/omni_ar_scheduler.py::_slice_sampled_logprobs`、`update_from_output` | | stateful async chunk、full-payload input、KV cleanup | SCHED-5b/5c | `core/sched/omni_generation_scheduler.py`、`omni_scheduling_coordinator.py`、`omni_ar_scheduler.py::_free_request` | | KV extraction wait、connector acknowledgement、partial interval cleanup | SCHED-5h | `core/sched/omni_ar_scheduler.py::{_free_request,update_from_output}` → scheduler stats → orchestrator metrics | +| consumed chunk validation failure、receive ledger、live request termination | SCHED-5i | `core/sched/omni_scheduler_mixin.py::_process_chunk_receive_failures` → `chunk_transfer_adapter.py` | | async stop、streaming update、stale token fence、next duplex unit 被吞 | SCHED-6a | `core/sched/omni_ar_scheduler.py::{_handle_stopped_request,update_from_output}` → `test_omni_ar_scheduler_streaming.py` | 若描述只写模型症状，先从模型 owner 找到 payload producer/consumer；只有实际断点落在调度、 @@ -271,6 +272,18 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 禁止：把任意已完成请求重新打开；对脱离所有 live queue、可能等待 deferred block free 的终态请求调用释放；因 request 在本轮进入时是 `WAITING_FOR_CHUNK` 就撤销其同轮 requeue；全局清空 running 中无关的 resumable segment；重复消费单遍 request-id iterator，或因非流式 skipped 请求错误减少 streaming counter。 - 验收：AR 与 generation scheduler 均覆盖 hidden、`running`、`waiting`、`skipped_waiting`、脱离队列及无关 resumable 请求；另覆盖 connector-fed duplex receiver 从 segment stop 同轮转回 `WAITING`，断言它保留在 waiting queue、离开 skipped queue、流式等待计数递减且 segment-finished 标记清除；sender-only duplex 与 connector-fed turn-mode controls 必须仍 parked，同时也清除标记。首次 finish 恰好释放、第二次无操作，并验证 generator request-id 输入。PR 作者报告的 H100 E2E 与 574-pass CPU/config suite 是提交时证据，不是当前环境或跨硬件保证。^[PR #6089] ^[PR #6360] ^[PR #6680] +## SCHED-5i — consumed connector contract failure 必须在 scheduler thread 终止 live request + +- 触发：receiver 已消费 async-chunk，但 prompt/window validation 失败，后台线程不能安全修改 request + status 或释放 scheduler ownership。 +- 强制：receiver 只按 internal request ID 记录首次失败原因；scheduler 每轮原子 drain ledger，仅对仍在 + `self.requests` 的 ID 调 `finish_requests(..., FINISHED_ERROR)`。已离开或被同 ID 新 generation 替换的 + request 不得由 stale failure 终止；错误日志保留 request 与原因。 +- 禁止：在 recv thread 直接结束 request；重试已消费 chunk；将 stale ID 当作 live request；因单个 + malformed duplex condition 阻断同批健康请求。 +- 验收：覆盖 live、已完成/abort、同 ID replacement、重复 failure 与同批健康 request，断言只终止 + 当前 owner 且 ledger 只消费一次。^[PR #6626] + ## SCHED-6a — async discard 的计数单位必须与 stale drain 一致 - 触发：上游 scheduler 改动异步占位、stale output 或 streaming segment replacement。 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ffb681a48..5295d9950 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3adee400` 验证存在 +- 源码校验：以上路径均已在 `main @ 87736a3c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b590a8d7b..ffad821b7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3adee400`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 87736a3c`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 676a35fa \ - --to 3adee400 \ + --from 3adee400 \ + --to 87736a3c \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md index 4e638157b..3bec887a4 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", "PR #6767", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/deploy/minicpmo_4_5_2gpu.yaml, vllm_omni/deploy/minicpmo_4_5_3gpu.yaml, vllm_omni/deploy/minicpmo_4_5_8x4090.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] +sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", "PR #6626", "PR #6767", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/deploy/minicpmo_4_5_2gpu.yaml, vllm_omni/deploy/minicpmo_4_5_3gpu.yaml, vllm_omni/deploy/minicpmo_4_5_8x4090.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] confidence: high --- @@ -69,4 +69,25 @@ confidence: high - 强制：client 以累计 appended PCM 的 unit boundary 绑定 frame `k`：首个 unit 在约 `1030 ms`（`first_chunk_ms=1035` 对齐 160-sample hop 后），后续每 `1000 ms`；frame 只能附在**关闭 unit `k`**的 append，不能在早期 append 暂存或跨 append carry。视频短于音频时保持最后一帧，单张 still 同样重复。`stack_frames=2` 是 previous+current 的两张图；整个 append 的 `frame_list` 都属于它关闭的一个 unit，顺序为 ``、每张各自 `` + 64 vision embeddings + ``、再 audio。vision hidden states 在拼入 token embedding 前转为 token-embedding dtype。 - 拒绝：携 frame 但没有关闭 model unit 的 append 必须显式失败，且不推进 `audio_chunk_idx`；不能只丢帧后继续音频。Realtime wire 每 append 最多两张图，stacked composite 不改变一秒 audio cadence。视频 demo 同时传 `--input-wav` 和 `--input-video` 时，外部 WAV 覆盖视频 soundtrack 且决定 frame timeline/duration。 - 验收：覆盖 frame 0 不早于约 1030 ms、每个 frame `k` 绑定其 closing append、无跨 append carry、short/still hold-last、`[f0]`/`[f0,f1]`/`[f1,f2]` stacked 序列、双图 marker/128 embedding 布局、no-unit rejection 不推进 audio index、dtype 对齐，以及 silent video + external WAV。Daily-Omni pack-mode 测试只覆盖数据包装边界，不能替代 Realtime session 或 Stage 0 验证。^[PR #6404] + +## MCPMO-4g — native duplex Stage 1 rollover 只重算 Talker context + +- 触发：修改 native-duplex Talker 长上下文、`attention_type=sliding_recompute`、async-chunk + prompt replacement、codec repetition penalty 或 receive-side request registration。 +- 强制：sliding recompute 只由 Stage 1 Talker checkpoint/config 的显式 + `attention_type="sliding_recompute"` 启用；native duplex 本身不能覆盖 checkpoint policy。effective + context limit 取正的 vLLM `max_model_len` 与 TTS `max_position_embeddings` 的较小值；每个新 condition + 必须声明正整数 `next_stage_generation_tokens=26`，fresh/sliding prompt 加 reserve 都不得越界。 +- 强制：rollover 仅保留前一 condition、该段已确认且不含 EOS/非法值的至多 25 个 codec ids、当前 + condition；condition sequence 必须单调连续。replacement 在 scheduler thread 原子清空旧 output/KV + prompt ownership、重建 block hashes 并回到 admission；model 以相同 sequence 验证/replay frozen + embeddings，repetition penalty 只继承最近 16 个 codes。turn boundary 不得跨 turn recompute。 +- 禁止：让 Stage 0 或非显式 Talker 进入 sliding window；从 placeholder/未确认 token 构造 rollover； + 累积全部 codec history；在 receiver thread 直接改 scheduler token counters；异常后跳过已消费 chunk + 继续运行。任一缺失 metadata、跳号、超长、EOS/非法 codec 或 prompt-length mismatch 必须记录为 + receive failure，并由 scheduler 将仍存活 request 置为 `FINISHED_ERROR`。 +- 验收：覆盖 first condition、连续 rollover、same-condition replay、turn reset、limit/reserve 边界、 + confirmed-placeholder 裁剪、EOS/非法/跳号/缺 metadata 拒绝、latest-16 penalty 与 cleanup/ID reuse； + scheduler 必须只处理当前 live request。目标合入的 CPU/unit tests 证明状态机与 fencing；PR 最终 + real-weight E2E 未全绿，不能据此声称音频质量、长会话稳定性或跨硬件支持。^[PR #6626] - 已知边界：Stage 0 vision encode 失败虽返回 failed prefill 且不推进 `audio_chunk_idx`，但 `preprocess()` 仍可能 fallback 到原 input ids/embeddings；保留的 scheduler span/KV 或 `num_computed_tokens` 是否推进尚未 abort/rollback 证明。另一个 fast/malformed client 的 pending-frame queue 仍未封顶。两者都是未合入 follow-up，不得写成此 frame-binding 修复已解决。此前 NPUGraph / soft-interrupt 范围已从本 PR 移除。^[PR #6404] From 564fc531608c1152d8639f2d3e2ead49c07af04f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:55:40 +0800 Subject: [PATCH 382/465] knowledge(vllm-omni): sync 73893e73 (PR #6773) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-upstream-compat.md | 16 ++++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 26 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e21ad234a..8be78ca3e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3adee400a03454d483a7c07018f4c65a5b8ab8f9 + previous_audited_sha: 87736a3cc50a96b5c6e83e631c7b4d3a256fcae0 audited_ref: main - audited_sha: 87736a3cc50a96b5c6e83e631c7b4d3a256fcae0 + audited_sha: 73893e73eace994c72daee41f5398065966ab793 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4b2a6a90f..ee4213aae 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -87736a3cc50a96b5c6e83e631c7b4d3a256fcae0`. The canonical machine baseline +73893e73eace994c72daee41f5398065966ab793`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 68cfe5cb3..25a6542fc 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 87736a3c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 73893e73` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8e64c81f6..2826084df 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 87736a3c` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 73893e73` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b8f887360..894414a3b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 87736a3c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 73893e73` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7ba219d19..97ff62888 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 87736a3c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 73893e73` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d05b1beca..2fb5a23f7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 87736a3c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 73893e73` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5295d9950..8c75fea72 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 87736a3c` 验证存在 +- 源码校验：以上路径均已在 `main @ 73893e73` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md index 5c99ec712..4e7e3d884 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md @@ -1,10 +1,10 @@ --- title: "Serving upstream 兼容规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, "PR #5036", "PR #6642"] +sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, "PR #5036", "PR #6642", "PR #6773", vllm_omni/config/endpoint_policy.py] confidence: high --- @@ -40,3 +40,15 @@ confidence: high - 强制：使用上游 `is_runai_obj_uri` 作为唯一对象存储判断，在 Hugging Face/ModelScope 预下载前原样返回对象存储 URI；本地路径与普通 HF repo id 保持既有分支，并将后续配置读取交给配置层的本地物化逻辑。 - 禁止：维护与 vLLM streamer 分离的本地 scheme allowlist；将对象存储 URI 当作 HF repo id 下载；让 bucket 或组织路径段参与模型名称匹配。 - 验收：参数化 mock 测试覆盖 `s3://`、`gs://` 及上游支持的 `az://`，断言 HF 下载未调用且返回值未改变；同时覆盖本地路径、HF repo id 和欺骗性 bucket/组织段匹配，确认入口语义没有回归。resolved HF cache snapshot 的 repo-name recovery 已随 #6642 的 revert 不再是当前合同。^[PR #5036] ^[PR #6642] + +## SERV-7d — upstream error-response helper 必须按能力边界兼容两种布局 + +- 触发：pinned vLLM 移动 `create_error_response`，或 endpoint policy / API server 增加直接 caller。 +- 强制：两个 caller 都先从新版 package owner `vllm.entrypoints.serve` 导入；只有该 import 抛 + `ImportError` 时才回退旧布局 `vllm.entrypoints.serve.utils.error_response`。两处必须保持同一顺序， + 使 endpoint rejection 与 engine error 都继续产生 upstream structured response。 +- 禁止：捕获任意 `Exception` 后伪装成版本兼容；在不同 caller 固定不同路径；因 import 成功就宣称 + 跨版本 API/JSON parity。该 shim 只覆盖这一个 symbol 的两种已知位置。 +- 验收：分别在仅提供新版 re-export、仅提供旧 module、两者都缺失的 import fixture 下加载两个 + caller，并验证 400/error response shape。PR #6773 没有新增自动化回归或硬件执行证据；其 diff + 只证明双 caller 的 import fallback，不能扩展为完整 vLLM 版本兼容声明。^[PR #6773] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ffad821b7..0a63c45e8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 87736a3c`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 73893e73`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 3adee400 \ - --to 87736a3c \ + --from 87736a3c \ + --to 73893e73 \ --repo \ --mode report-only ``` From 4b9578103e7e682fc5684ad37e7f426beccd2fbd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 03:59:20 +0800 Subject: [PATCH 383/465] knowledge(vllm-omni): sync 9a88d2d2 (PR #6749) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/architecture.md | 9 +++++++-- .../diffusion/rules-output-lifecycle.md | 19 ++++++++++++++++++- .../vllm-omni/components/diffusion/rules.md | 1 + .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 12 files changed, 38 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8be78ca3e..1fb3eea6b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 87736a3cc50a96b5c6e83e631c7b4d3a256fcae0 + previous_audited_sha: 73893e73eace994c72daee41f5398065966ab793 audited_ref: main - audited_sha: 73893e73eace994c72daee41f5398065966ab793 + audited_sha: 9a88d2d21d4b3a97f7e7a5d2d751328ffda08379 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ee4213aae..4c49ac74b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -73893e73eace994c72daee41f5398065966ab793`. The canonical machine baseline +9a88d2d21d4b3a97f7e7a5d2d751328ffda08379`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 25a6542fc..aa2955981 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 73893e73` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 9a88d2d2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2826084df..b80b2daba 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 73893e73` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 9a88d2d2` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/architecture.md b/knowledge/repos/vllm-omni/components/diffusion/architecture.md index 336e1cc4c..2c42c5a59 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/architecture.md +++ b/knowledge/repos/vllm-omni/components/diffusion/architecture.md @@ -1,10 +1,10 @@ --- title: "Diffusion 共享架构" created: 2026-07-10 -updated: 2026-07-16 +updated: 2026-09-05 type: architecture tags: [vllm-omni, components, diffusion] -sources: [vllm_omni/diffusion/] +sources: [vllm_omni/diffusion/, "PR #6749"] --- # Diffusion 共享架构 @@ -32,4 +32,9 @@ sources: [vllm_omni/diffusion/] 2. 沿配置、runner、pipeline、denoise loop 和输出逐层查证。 3. 如果最终只在一个模型的 pipeline 或 checkpoint 上复现，把正文放回模型目录，并从这里链接。 +执行身份与 checkpoint 身份必须分开追踪：custom pipeline、Diffusers adapter 与 native pipeline +决定实际 executor 和 hook；checkpoint `model_class_name` 则继续提供 modality/capability metadata。 +不要因 `diffusion_load_format=diffusers` 就覆盖 native metadata 或绕过 custom-pipeline precedence； +具体审查合同见 [DIFF-1aa](rules-output-lifecycle.md)。 + 源码会变化，具体类名和路径在改代码前必须以目标仓库当前版本为准。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index bf99b8a1c..a289710cb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,12 +4,29 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499"] +sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py] confidence: high --- # Diffusion output 与 multiprocess runtime 规则 +## DIFF-1aa — Diffusers execution backend 不得冒充 native checkpoint identity + +- 触发：修改 `diffusion_load_format`、custom pipeline 选择、checkpoint `model_class_name`、 + pre/post-process hook、request batching 能力或 frame interpolation。 +- 强制：`uses_diffusers_adapter()` 只表示实际 execution backend；存在 + `custom_pipeline_args` 时 custom pipeline 优先并令其为 false，即使 load format 是 Diffusers。 + checkpoint 的 native `model_class_name` 继续承载 modality/capability metadata，但 effective + Diffusers adapter 不运行 native pre/post hooks。batch capability 必须从实际 executor 派生：先 + custom pipeline，再 `DiffusersAdapterPipeline`，否则才查 native class。Diffusers adapter 收到 + frame interpolation 必须 fail fast，因为它交付的 frame 已完成 post-process。 +- 禁止：从 checkpoint metadata 或 load format 单独推导 execution backend、native hooks 或 batching； + custom pipeline 已接管时仍套 adapter hook；对 adapter output 静默二次 interpolation。 +- 验收：组合覆盖 native、Diffusers adapter、Diffusers load format + custom pipeline 三类配置，断言 + class identity 保留、hook 选择和 batch capability 跟随实际 executor；另验证所有 Diffusers pipeline + 对 interpolation 抛出明确错误。PR #6749 的自动化测试覆盖这些选择边界，但不证明任意第三方 + custom pipeline 或未来 Diffusers 版本兼容。^[PR #6749] + ## DIFF-1d — async output 的 compute 与 output-ready 必须分成两个可观察阶段 - 触发：把 diffusion D2H/SHM packing 移到 worker 后台线程，或修改 result pump、`collective_rpc`、 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index a9b41089b..1026e18e4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -37,6 +37,7 @@ confidence: high | worker title、log prefix、DP/PP/SP/CFG/TP/FS/RP/EP local rank | [worker observability](rules-worker-observability.md)：`DIFF-4w` | `diffusion_worker.py::_setup_diffusion_worker_proc_title_and_log_prefix` → `distributed/parallel_state.py` group getters | | image/diffusion service time、forward time、missing-versus-zero、per-step metric | [`metrics-evidence`](rules-metrics.md)：`DIFF-5a`, `DIFF-5b` | output formatter/sampling metadata → stage stats aggregation → serving Prometheus emission | | image/video/audio/action final type、registry alias、endpoint capability、formatter | [`output-runtime`](rules-output-lifecycle.md#diff-1s-diffusion-output-type-必须从模型声明闭合到-topology-与-formatter)：`DIFF-1s` | `model_metadata.py` → default/multi-stage config → video capability → `output_formatter.py` | +| Diffusers/custom/native executor identity、hook、batch capability、frame interpolation | [`output-runtime`](rules-output-lifecycle.md#diff-1aa-diffusers-execution-backend-不得冒充-native-checkpoint-identity)：`DIFF-1aa` | `data.py::uses_diffusers_adapter` → `diffusion_engine.py::supports_request_batch` → registry hook / adapter sampling validation | | 审查组 | 什么时候触发 | 规则 ID | |---|---|---| diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 894414a3b..cfc0c8479 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 73893e73` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 9a88d2d2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 97ff62888..25714b7b1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 73893e73` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 9a88d2d2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2fb5a23f7..a704270a8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 73893e73` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 9a88d2d2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8c75fea72..f874e3cde 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 73893e73` 验证存在 +- 源码校验：以上路径均已在 `main @ 9a88d2d2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0a63c45e8..242e4ab9b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 73893e73`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 9a88d2d2`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 87736a3c \ - --to 73893e73 \ + --from 73893e73 \ + --to 9a88d2d2 \ --repo \ --mode report-only ``` From fb00cc6d05cf34715630bcafd2202830fc794e40 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:10:34 +0800 Subject: [PATCH 384/465] knowledge(vllm-omni): sync 21dd4b33 (PR #6529) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 3 ++- .../vllm-omni/components/distributed/rules.md | 17 ++++++++++++- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/scheduler/rules.md | 20 +++++++++++++-- .../vllm-omni/components/serving/_index.md | 3 ++- .../serving/rules-duplex-orchestration.md | 25 +++++++++++++++++++ .../vllm-omni/components/serving/rules.md | 4 +-- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../models/minicpm-o-4-5/rules-duplex.md | 20 ++++++++++++++- .../vllm-omni/models/minicpm-o-4-5/rules.md | 3 ++- 15 files changed, 96 insertions(+), 19 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-duplex-orchestration.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1fb3eea6b..e1483dc39 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 73893e73eace994c72daee41f5398065966ab793 + previous_audited_sha: 9a88d2d21d4b3a97f7e7a5d2d751328ffda08379 audited_ref: main - audited_sha: 9a88d2d21d4b3a97f7e7a5d2d751328ffda08379 + audited_sha: 21dd4b33ac46e71479e839a9de3ed1b801046296 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4c49ac74b..6d33b8781 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -9a88d2d21d4b3a97f7e7a5d2d751328ffda08379`. The canonical machine baseline +21dd4b33ac46e71479e839a9de3ed1b801046296`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index aa2955981..b8af1aba9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 9a88d2d2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 21dd4b33` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b80b2daba..e0416c718 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 9a88d2d2` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 21dd4b33` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index cfc0c8479..7bd9d232f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 9a88d2d2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 21dd4b33` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） @@ -33,6 +33,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo | PR 描述信号 | 第一批源码 | 已有知识 | |---|---|---| | connector backend、put/get、跨 stage 数据损坏 | `distributed/omni_connectors/connectors/base.py::OmniConnectorBase`；目标 backend 的 `put` / `get` | architecture | +| async sender boundary、segment generation、chunk dedup watermark | `distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py::save_async` | DIST-1g/1j | | Mooncake、RDMA/TCP fallback、object decode | `connectors/mooncake_transfer_engine_connector.py::MooncakeTransferEngineConnector` | connector pitfalls | | KV transfer、connector lifecycle | `distributed/omni_connectors/kv_transfer_manager.py::OmniKVTransferManager` | architecture | | coordinator、replica selection、负载均衡 | `distributed/omni_coordinator/load_balancer.py::LoadBalancer` 及具体 balancer | architecture | diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 87164a783..309c03339 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-05 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626"] +sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626", "PR #6529"] confidence: high --- @@ -104,6 +104,21 @@ confidence: high - 验收：覆盖 cleanup-during-get、same-ID re-registration、late success/error、重复 register/cleanup、 scheduler live-ID replacement 与 invalid payload ledger drain；证明旧 entry 不影响新 request。^[PR #6626] +## DIST-1j — sender dedup watermark 必须按 generation/segment 限定 + +- 触发：async-chunk sender dedup、segment terminal task、background cleanup 或同 request ID 的 + streaming segment replacement。 +- 强制：`save_async` 将 request 的 segment generation 冻结进 queued task；小于当前 generation 的 + late save 必须拒绝。terminal task 在后台 sender 执行前先把 generation 推进到下一段并清空旧 + chunk watermark，非 terminal preemption 才用 confirmed-token watermark 去重。request 最终 cleanup + 同时回收 generation 和 chunk state。 +- 禁止：只按裸 request ID 判断 late task 有效；让旧 terminal boundary 因旧 chunk count 被误拒； + background completion 无条件删除下一段 watermark；把未 ACK 的 serving response snapshot描述成 + 此 sender generation 已给出有界 retention。 +- 验收：覆盖 old terminal 排队后 new segment 先开始、late old save、boundary 在 chunk counter reset + 后仍入队，以及普通 dedup/preemption；断言 old payload 不复活且 new generation/watermark 保留。 + ^[PR #6529] + ## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 - 触发：上游 MoE factory/oracle 或 `init_model_parallel_group` 增加 all-to-all manager 要求。 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 25714b7b1..0cd4fbd21 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 9a88d2d2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 21dd4b33` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a704270a8..e87d39056 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 9a88d2d2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 21dd4b33` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 9fafe2ab2..309827e6e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-05 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/core/sched/test_omni_scheduler_mixin_timeouts.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619", "PR #6680", "PR #6626"] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/core/sched/test_omni_scheduler_mixin_timeouts.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619", "PR #6680", "PR #6626", "PR #6529", tests/core/sched/test_omni_ar_scheduler_aborted_queue_sweep.py, tests/core/sched/test_omni_scheduler_streaming_input_counter.py, tests/core/sched/test_omni_sched_deferred_free_fence.py] --- # Scheduler 规则 @@ -28,7 +28,7 @@ PR 描述先命中下表，再打开对应规则组和首批源码；changed fil | stateful async chunk、full-payload input、KV cleanup | SCHED-5b/5c | `core/sched/omni_generation_scheduler.py`、`omni_scheduling_coordinator.py`、`omni_ar_scheduler.py::_free_request` | | KV extraction wait、connector acknowledgement、partial interval cleanup | SCHED-5h | `core/sched/omni_ar_scheduler.py::{_free_request,update_from_output}` → scheduler stats → orchestrator metrics | | consumed chunk validation failure、receive ledger、live request termination | SCHED-5i | `core/sched/omni_scheduler_mixin.py::_process_chunk_receive_failures` → `chunk_transfer_adapter.py` | -| async stop、streaming update、stale token fence、next duplex unit 被吞 | SCHED-6a | `core/sched/omni_ar_scheduler.py::{_handle_stopped_request,update_from_output}` → `test_omni_ar_scheduler_streaming.py` | +| async stop、streaming update、scheduled/async stale token fence、abort queue/counter drift | SCHED-6a/6f | `core/sched/omni_ar_scheduler.py::{_handle_stopped_request,update_from_output}` → scheduler mixin abort sweep/counter resync → streaming/stale tests | 若描述只写模型症状，先从模型 owner 找到 payload producer/consumer；只有实际断点落在调度、 prefix cache 或 copy lifetime 时才把 Scheduler 加为 owner。 @@ -331,3 +331,19 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 - 强制：在 `_handle_stopped_request` 及任何会重置 computed-token 状态的 transition 前，从 chunk adapter 取得 confirmed computed-token watermark；将该冻结值显式传给 `save_async`。后续 segment 只能建立自己的 watermark，不能让旧边界任务读取已重置 request。 - 禁止：在 request replacement 后重新计算旧 segment 的 send watermark；以当前 `num_computed_tokens` 推断已确认旧边界；把模型专用 Code2Wav batching/wait policy 塞进通用 generation scheduler。 - 验收：模拟 stop handler 将同一 request 重置为新 segment，断言 queued old boundary 使用 transition 前的 confirmed token 数；并验证 generation scheduler 没有新增 MiniCPM/Code2Wav-specific coalescing state。^[PR #6021] + +## SCHED-6f — stale output 必须同时耗尽 scheduled 与 async 两类计数 + +- 触发：streaming segment replacement 后收到 late model frame，或修改 abort queue cleanup、 + streaming-input admission counter 与 segment-boundary handoff。 +- 强制：同一迟到 frame 必须联合结算 scheduled-token stale window 与 + `async_tokens_to_discard`；前者按 scheduled-token accounting，后者按该 frame 的 generated-token + 数递减，任一 fence 命中就不得 append/emit。处理 pending input 前后都从 + `waiting`、`skipped_waiting` 与 `running` 清除 `FINISHED_ABORTED`，随后按 live queue ownership + 重算 `num_waiting_for_streaming_input`。 +- 禁止：两个 stale domain 只消费一个；只清普通 waiting/running 而遗留 hidden skipped queue； + 用增减猜 streaming counter；把 unacknowledged response snapshot 当作有界 lifetime，或把 + sliding-recompute/后续 playback ACK 修复归因于本提交。 +- 验收：覆盖一个 frame 同时命中两种 fence、仅命中任一 fence、三类 queue 的 abort sweep、 + pending-input 前后 abort，以及 live-owner counter drift 修复；断言旧 frame 不进入 output， + 两个 counter 恰好归零且新 segment 首帧保留。^[PR #6529] diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f874e3cde..27a3b4db7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 9a88d2d2` 验证存在 +- 源码校验：以上路径均已在 `main @ 21dd4b33` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -34,6 +34,7 @@ sources: [] | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | | engine startup/shutdown、stage lifecycle、acknowledged abort 的 shutdown race、full-duplex、CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | +| duplex stage-keyed segment state、跨 stage boundary/metadata 隔离 | [duplex 编排规则](rules-duplex-orchestration.md) | | acknowledged abort 的 RPC ACK、queue/router shutdown race、cancellation cleanup 或 backpressure/timeout 保持 | [acknowledged abort 生命周期规则](rules-abort-lifecycle.md) | | assembled FastAPI route 覆盖、method/path ownership 或 request-body forwarding | [app assembly 规则](rules-app-assembly.md) | | stage config→EngineArgs projection、explicit `devices`、TP/local-DP/PP、replica layout 或 worker 创建前的 layout guard | [stage 启动与设备布局规则](rules-stage-startup.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-duplex-orchestration.md b/knowledge/repos/vllm-omni/components/serving/rules-duplex-orchestration.md new file mode 100644 index 000000000..c3ba34800 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-duplex-orchestration.md @@ -0,0 +1,25 @@ +--- +title: "Serving duplex 编排规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #6529", vllm_omni/engine/orchestrator.py, tests/engine/test_orchestrator_segment_stage_keying.py, tests/engine/test_orchestrator_stage_input_bridge.py] +confidence: high +--- + +# Serving duplex 编排规则 + +## SERV-6g — orchestrator streaming segment state 必须按 stage key 隔离 + +- 触发：orchestrator streaming state、stage metrics、final-output routing、next-stage bridge 或 duplex + output context 修改。 +- 强制：每个 stage 独立拥有 `finished`、`token_ids` 与 `output_metadata`；写入和所有 consumer 都以 + 显式 `stage_id` 选择该 slot，未上报的 stage 返回 fresh empty state，不能共享 mutable metadata。 + `new_prompt_len_snapshot` 仍是 request-level，因为它只由不按 stage 读取的 input-bridge context + 消费。 +- 禁止：用 request-global flat segment slot 让 diffusion/raw-output poll 覆盖另一 stage；调用 duplex + context 时省略 stage ID；把 request-level prompt-length snapshot 误迁入 stage slot。 +- 验收：多 stage 交错上报 boundary，逐 stage 验证 metrics、final-output、bridge 与 duplex context + 只见自身 token/metadata；覆盖 missing-stage fresh state 与不会经过 raw-output loop 的 diffusion + branch。^[PR #6529] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index b15789bbc..265054ad0 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_api_server_guards.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288", "PR #6622", "PR #6499"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_api_server_guards.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288", "PR #6622", "PR #6499", "PR #6529"] confidence: high --- @@ -47,7 +47,7 @@ confidence: high | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 见 engine lifecycle；`SERV-5s` 见 [app assembly](rules-app-assembly.md) | | `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup/shutdown race、streaming raw terminal、event-driven orchestration、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5r`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）、[SERV-5t](rules-abort-lifecycle.md#serv-5t-acknowledged-abort-在-shutdown-边界只忽略已识别的-transport-closure)，`SERV-5f` | -| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion lifecycle | `SERV-6a`–`SERV-6d`（见 [engine 生命周期规则](rules-engine-lifecycle.md)） | +| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion 与 stage-keyed segment lifecycle | `SERV-6a`–`SERV-6f` 见 [engine 生命周期规则](rules-engine-lifecycle.md)；`SERV-6g` 见 [duplex 编排规则](rules-duplex-orchestration.md) | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds、transform artifact ownership | `SERV-4a`–`4h`, `SERV-4l`–`4q`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 242e4ab9b..b2d5da5dd 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 9a88d2d2`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 21dd4b33`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 73893e73 \ - --to 9a88d2d2 \ + --from 9a88d2d2 \ + --to 21dd4b33 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md index 3bec887a4..931dbe238 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", "PR #6626", "PR #6767", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/deploy/minicpmo_4_5_2gpu.yaml, vllm_omni/deploy/minicpmo_4_5_3gpu.yaml, vllm_omni/deploy/minicpmo_4_5_8x4090.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py] +sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", "PR #6626", "PR #6767", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/deploy/minicpmo_4_5_2gpu.yaml, vllm_omni/deploy/minicpmo_4_5_3gpu.yaml, vllm_omni/deploy/minicpmo_4_5_8x4090.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py, "PR #6529", vllm_omni/experimental/fullduplex/openai/protocol.py, vllm_omni/experimental/fullduplex/openai/realtime_state.py, tests/entrypoints/openai/test_duplex_protocol.py] confidence: high --- @@ -91,3 +91,21 @@ confidence: high scheduler 必须只处理当前 live request。目标合入的 CPU/unit tests 证明状态机与 fencing；PR 最终 real-weight E2E 未全绿，不能据此声称音频质量、长会话稳定性或跨硬件支持。^[PR #6626] - 已知边界：Stage 0 vision encode 失败虽返回 failed prefill 且不推进 `audio_chunk_idx`，但 `preprocess()` 仍可能 fallback 到原 input ids/embeddings；保留的 scheduler span/KV 或 `num_computed_tokens` 是否推进尚未 abort/rollback 证明。另一个 fast/malformed client 的 pending-frame queue 仍未封顶。两者都是未合入 follow-up，不得写成此 frame-binding 修复已解决。此前 NPUGraph / soft-interrupt 范围已从本 PR 移除。^[PR #6404] + +## MCPMO-4h — response/playback history 必须由回答它的 input commit 所有 + +- 触发：native duplex response create/end、input commit、playback ACK、conversation item + register/delete/truncate 或 auto-response terminal closure。 +- 强制：active response 绑定它回答的 input commit sequence；如果 response 在 user-speaking 阶段创建， + 则绑定紧随其后的真实 commit。response-scoped snapshot 以 response ID 保存 assistant text、audio + text marks 与该 commit sequence。ACK 的 item 必须精确为 `item_`、确实属于已知 + assistant response，且不能晚于后续 input commit；mismatch/not-found/too-late 分别返回 typed error。 + register/delete/truncate 只消费并清理 matching response snapshot、pending marks/commit metadata。 + auto-response safety close 即使已没有 active response，也必须完成对应 model turn。 +- 禁止：用全局 last-assistant buffer 让晚到 ACK/truncate 绑定到另一 response；把前一 response history + 提交到下一 input；用 nonmatching item 清理 snapshot；把未 ACK/delete 的 snapshot 宣称已有有界 + retention。 +- 验收：覆盖 response-before/after-input commit、ACK identity/existence/late ordering、两个 response + 交错的 register/delete/truncate、pending audio marks、no-active auto close 与下一 fenced append；断言 + typed error 无副作用且只释放目标 snapshot。当前实现的 snapshots 在 ACK/delete 前仍可能无界。 + ^[PR #6529] diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 6f2b81cec..0c5f8a995 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757"] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757", "PR #6529", vllm_omni/experimental/fullduplex/openai/protocol.py] confidence: high --- @@ -30,6 +30,7 @@ confidence: high | instructions/native-mode update、first-append reservation、context lock | [`MCPMO-4b`](rules-duplex.md#mcpmo-4b-native-duplex-session-update-必须如实拒绝不可重建的上下文) | `experimental/fullduplex/minicpmo45/{adapter,session}.py` → runtime adapter/session runner | | duplex Talker chunk、runtime `min_tokens`、chat sampling isolation | [`MCPMO-4c`](rules-duplex.md#mcpmo-4c-native-duplex-talker-必须按-generate_chunk-预算终止) | `experimental/fullduplex/minicpmo45/adapter.py` → stage 1 sampling → Talker `generate_chunk` | | shipping YAML、duplex/chat 共服、session capacity、async scheduler | [`MCPMO-4e`](rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → config merge → realtime/chat entrypoints | +| playback ACK、response-owned assistant history、下一 input 后的 late ACK | [`MCPMO-4h`](rules-duplex.md#mcpmo-4h-responseplayback-history-必须由回答它的-input-commit-所有) | `experimental/fullduplex/openai/{protocol,serving}.py` → response snapshot / conversation item lifecycle | | Daily-Omni/Seed-TTS accuracy、simplex/duplex perf、TTFT/TTFP/RTF | MCPMO-5a | `tests/e2e/accuracy/minicpmo_4_5/`、`tests/dfx/perf/tests/test_minicpmo_4_5*.json` → `run_benchmark.py` | | online serving CI、async/sync chunk matrix、duplex fixture、local ref audio | MCPMO-5b | `tests/e2e/online_serving/test_minicpmo_4_5*.py` → `helpers/minicpmo_4_5_duplex.py` | From 6eb70631e92e644a8cb49944348339f447240c5b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:15:28 +0800 Subject: [PATCH 385/465] knowledge(vllm-omni): sync 99a2ae1f (PR #6772) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minicpm-o-4-5/rules.md | 14 ++++++++++++-- 10 files changed, 24 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e1483dc39..9092a638e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 9a88d2d21d4b3a97f7e7a5d2d751328ffda08379 + previous_audited_sha: 21dd4b33ac46e71479e839a9de3ed1b801046296 audited_ref: main - audited_sha: 21dd4b33ac46e71479e839a9de3ed1b801046296 + audited_sha: 99a2ae1fc8637e63a5736a0da68c087caa5f633b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 6d33b8781..b25fd5fca 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -21dd4b33ac46e71479e839a9de3ed1b801046296`. The canonical machine baseline +99a2ae1fc8637e63a5736a0da68c087caa5f633b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b8af1aba9..c8ace45da 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 21dd4b33` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 99a2ae1f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e0416c718..55a0a47cd 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 21dd4b33` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 99a2ae1f` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7bd9d232f..06a91e680 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 21dd4b33` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 99a2ae1f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0cd4fbd21..b1a424e89 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 21dd4b33` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 99a2ae1f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e87d39056..7d2783b95 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 21dd4b33` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 99a2ae1f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 27a3b4db7..086f86264 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 21dd4b33` 验证存在 +- 源码校验：以上路径均已在 `main @ 99a2ae1f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b2d5da5dd..03c1398da 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 21dd4b33`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 99a2ae1f`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 9a88d2d2 \ - --to 21dd4b33 \ + --from 21dd4b33 \ + --to 99a2ae1f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 0c5f8a995..3ab516fce 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757", "PR #6529", vllm_omni/experimental/fullduplex/openai/protocol.py] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757", "PR #6529", "PR #6772", vllm_omni/experimental/fullduplex/openai/protocol.py] confidence: high --- @@ -32,7 +32,7 @@ confidence: high | shipping YAML、duplex/chat 共服、session capacity、async scheduler | [`MCPMO-4e`](rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → config merge → realtime/chat entrypoints | | playback ACK、response-owned assistant history、下一 input 后的 late ACK | [`MCPMO-4h`](rules-duplex.md#mcpmo-4h-responseplayback-history-必须由回答它的-input-commit-所有) | `experimental/fullduplex/openai/{protocol,serving}.py` → response snapshot / conversation item lifecycle | | Daily-Omni/Seed-TTS accuracy、simplex/duplex perf、TTFT/TTFP/RTF | MCPMO-5a | `tests/e2e/accuracy/minicpmo_4_5/`、`tests/dfx/perf/tests/test_minicpmo_4_5*.json` → `run_benchmark.py` | -| online serving CI、async/sync chunk matrix、duplex fixture、local ref audio | MCPMO-5b | `tests/e2e/online_serving/test_minicpmo_4_5*.py` → `helpers/minicpmo_4_5_duplex.py` | +| online serving CI、async/sync chunk matrix、duplex fixture、soft-interrupt event oracle、local ref audio | MCPMO-5b | `tests/e2e/online_serving/{run_minicpmo_realtime_duplex_soft_interrupt.py,test_minicpmo_realtime_duplex_drivers.py}` → `helpers/minicpmo_4_5_duplex.py` | ## MCPMO-1a — trust_remote_code 服从用户选择 @@ -285,5 +285,15 @@ confidence: high flat base-frame list；这是 CPU fixture input-shape contract，不改变 runtime camera stacking、H100 E2E 或模型行为。^[PR #6757] +- soft-interrupt 的 `response-required` artifact oracle 把“commit 前 response 后的 LISTEN”锚定为 + `input_audio_buffer.committed` **严格之前**最后一个 `response.done`，并要求该 LISTEN 严格落在 + 两者之间；不能用全局最后一个 done，因为 follow-up 可在 commit 后继续 drain residual model unit。 + `listen_after_last_done` 仍独立锚定全局最后一个 done，故 residual fixture 还必须在该 late done 后有 + LISTEN。`model-policy` 模式仅要求 commit 前已创建的完成 audio response，不采用该完整 sandwich。 + 验收用事件序列 `r1.done → LISTEN → r2.created → … → commit → r2.done → LISTEN`，断言 + `ok` 与两个 listen predicates；这只是 summarizer/test oracle，未改变 ACK、runtime interrupt、barge-in + detection 或生产响应时序。A100 的已知失败仍是 `enough_responses=false`（barge-in 漏检），不是本 PR + 修复范围；H100-class 测试计划也不是跨硬件实时性证明。^[PR #6772] ^[Issue #6716] + 共享 bridge/batch 规则见 [Model Executor rules](../../components/model-executor/rules.md)； 公开入口完整性见 [model adaptation guardrails](../../review/guides/model-adaptation-guardrails.md)。 From 74fa264c3d04ada99166f95e2c6f28bbce15fd55 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:23:07 +0800 Subject: [PATCH 386/465] knowledge(vllm-omni): sync 8be601ed (PR #6340) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 3 ++- .../rules-diffusion-parallel-transport.md | 18 ++++++++++++++++++ .../components/configuration/rules.md | 3 +++ .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/parallelism.md | 8 ++++++-- .../diffusion/rules-system-runtime.md | 12 +++++++++--- .../vllm-omni/components/diffusion/rules.md | 4 ++-- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 14 files changed, 51 insertions(+), 19 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/configuration/rules-diffusion-parallel-transport.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9092a638e..896f13d6b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 21dd4b33ac46e71479e839a9de3ed1b801046296 + previous_audited_sha: 99a2ae1fc8637e63a5736a0da68c087caa5f633b audited_ref: main - audited_sha: 99a2ae1fc8637e63a5736a0da68c087caa5f633b + audited_sha: 8be601ed7f2de17b524da47a5f6e383254743e22 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b25fd5fca..df6359528 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -99a2ae1fc8637e63a5736a0da68c087caa5f633b`. The canonical machine baseline +8be601ed7f2de17b524da47a5f6e383254743e22`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c8ace45da..76985273c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 99a2ae1f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8be601ed` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 @@ -38,6 +38,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] | 根据 PR 描述直达 strict schema、pipeline sampling constraints、deploy/topology、composable strategy 或显存配置的规则组与第一批源码 | [配置开发门禁与代码地图](rules.md) | | 对象存储 URI、HF cache snapshot 或空 `config.json` 的模型名称回退与 pipeline 路由 | [模型引用路由规则](rules-model-reference-routing.md) | | stage full-payload transport capability、topology-owned projection 或 deploy/CLI override rejection | [stage transport capability](rules-stage-transport.md) | +| diffusion Ulysses SymmMem transport flag 的 deploy/CLI/default-stage projection | [diffusion parallel transport rules](rules-diffusion-parallel-transport.md) | | 审计配置来源和多层加工 | [config audit](config-audit-plain-language.md) | | 设计配置归一化与 parity 验证 | [config normalization parity](config-normalization-parity.md) | | 查询 `Omni()` 初始化参数 | [Omni init args](omni-init-args.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-diffusion-parallel-transport.md b/knowledge/repos/vllm-omni/components/configuration/rules-diffusion-parallel-transport.md new file mode 100644 index 000000000..88b671e8d --- /dev/null +++ b/knowledge/repos/vllm-omni/components/configuration/rules-diffusion-parallel-transport.md @@ -0,0 +1,18 @@ +--- +title: "Diffusion parallel transport 配置规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, config, diffusion] +sources: ["PR #6340", vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/arg_utils.py, vllm_omni/entrypoints/cli/serve.py, tests/config/test_omni_config.py, tests/entrypoints/test_async_omni_diffusion_config.py, tests/entrypoints/test_serve.py] +confidence: high +--- + +# Diffusion parallel transport 配置规则 + +## VOMNI-CFG-1q — SymmMem Ulysses transport 开关必须完整投影且默认关闭 + +- 触发：新增/修改 diffusion 并行 transport flag、`StageDeployConfig`、`OmniStageDiffusionParallelConfig`、default diffusion stage 或 `serve` CLI。 +- 强制：`ulysses_a2a_permute` 在 deploy、structured parallel config、legacy/default-stage construction 和 CLI 都以同一 bool 字段传递；省略时最终值为 false，显式 true 必须到达 strict-Ulysses consumer，不能由 `ulysses_degree` 或其他 SP mode 隐式开启。 +- 禁止：只在 parser/dataclass 声明、在 default stage 或 deploy merge 丢失；以 truthiness 吞掉明确 false；把 `advanced_uaa`、Ring 或 AllGather 配置解释为已启用 SymmMem；将 JIT/capability failure 推迟到首个请求。 +- 验收：覆盖 CLI true/false、deploy true、structured/default-stage true 与 omitted false，断言最终 `parallel_config`；再由 DIFF-4x 覆盖 eligible consumer init 与不合资格组合不走 fused path。^[PR #6340] diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 04a6ebada..c3ccaed6c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -28,6 +28,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | multi-stage HF sub-config、stage quantization view、`hf_config_name` | `stage-model-config`：`VOMNI-CFG-1i` | pipeline stage declaration → `OmniModelConfig::{draw_hf_text_config,get_model_arch_config}` → vLLM quantization selection | | pipeline `sampling_constraints`、`StageConfig.to_omegaconf()`、runtime stage config | `stage-config-propagation`：`VOMNI-CFG-1o` | `merge_pipeline_deploy` → `StageConfig.to_omegaconf()` → `engine.stage_configs` | | stage transport、`requires_full_payload_input`、topology projection 或 override rejection | `stage-transport`：`VOMNI-CFG-1p` | `stage_config.py::{StagePipelineConfig,_build_engine_args}` → `omni_config.py::_build_model_config` → `engine/arg_utils.py::OmniEngineArgs` | +| SymmMem Ulysses transport | [VOMNI-CFG-1q](rules-diffusion-parallel-transport.md) + `DIFF-4x` | deploy → parallel config → Ulysses | | HF cache snapshot path、空 `config.json`、name-based pipeline fallback | [`model-reference-routing`](rules-model-reference-routing.md#conf-7a-模型引用解析必须物化对象存储配置并只从受控名称组件匹配)：`CONF-7a` | `config_factory.py::{_name_match_candidate,StageConfigFactory._try_infer_model_type}` → `pipeline_registry.py::OMNI_PIPELINES` → `tests/config/test_config_factory.py::TestNameMatchCandidateSnapshotPaths` | | Qwen3-TTS adaptive codec chunk keys、fixed-ramp precedence | [Qwen3-TTS `Q3TTS-3e`](../../models/qwen3-tts/rules.md#q3tts-3e-adaptive-ramp-是-host-side每段-opt-in-控制器不是-cuda-graph-计划) | `deploy/qwen3_tts{,_high_concurrency}.yaml` connector `extra` → `stage_input_processors/chunk_size_utils.py::parse_adaptive_config` | @@ -41,6 +42,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | | `stage-config-propagation` | pipeline sampling constraints、StageConfig serialization/runtime config | `VOMNI-CFG-1o` | | `stage-transport` | full-payload transport capability、topology projection、deploy/CLI override | `VOMNI-CFG-1p`，见 [stage transport capability](rules-stage-transport.md) | +| `diffusion-parallel-transport` | SymmMem Ulysses flag | [VOMNI-CFG-1q](rules-diffusion-parallel-transport.md) + `DIFF-4x` | | `model-reference-routing` | object-storage materialization、HF cache snapshot、name fallback | `CONF-7a`，见 [模型引用路由规则](rules-model-reference-routing.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `VOMNI-CFG-0a`, `VOMNI-CFG-0b` | @@ -133,6 +135,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR - 禁止：只在 dataclass 或默认 params 上保留约束、让 OmegaConf runtime config 静默丢字段，或以测试直接注入 constraints list 代替真实序列化路径。 - 验收：从 pipeline/deploy 合并走到 `StageConfig.to_omegaconf()` 和 runtime extraction，断言每 stage 的非默认 constraint 可读回；再由 serving 侧验证它到最终 sampling consumer 的效果。^[PR #6182] + ### CONF-1a — 多 stage 共卡时 diffusion stage 必须显式设 gpu_memory_utilization - 触发：多 stage 模型 stage 共卡时，分布式测试在模型加载期 CUDA OOM。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 55a0a47cd..6b89f42aa 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 99a2ae1f` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 8be601ed` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/parallelism.md b/knowledge/repos/vllm-omni/components/diffusion/parallelism.md index 0f118e7f5..dcb6022fb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/parallelism.md +++ b/knowledge/repos/vllm-omni/components/diffusion/parallelism.md @@ -1,10 +1,10 @@ --- title: "Diffusion 并行策略总览" created: 2026-07-16 -updated: 2026-07-16 +updated: 2026-09-05 type: guide tags: [vllm-omni, components, diffusion] -sources: [docs/design/feature/tensor_parallel.md, docs/design/feature/cfg_parallel.md, docs/design/feature/vae_parallel.md] +sources: [docs/design/feature/tensor_parallel.md, docs/design/feature/cfg_parallel.md, docs/design/feature/vae_parallel.md, "PR #6340", vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/attention/parallel/ulysses.py] --- # Diffusion 并行策略总览 @@ -22,5 +22,9 @@ sources: [docs/design/feature/tensor_parallel.md, docs/design/feature/cfg_parall （tp/dp/pp/ep/stage_replica 已接线；sp/cfg/vae_pp/hsdp 为保留位）见 [Configuration](../configuration/architecture.md)；stage 级 `tensor_parallel_size` 等 字段见 deploy schema。 +- strict Ulysses 可显式设置 `ulysses_a2a_permute=true` 使用 SymmMem permute-free A2A；它默认关闭， + advanced-UAA 与 AllGather 不走该 path；strict Ulysses+Ring 的 Ulysses leg 可用。workspace/capture/shutdown 与硬件 parity 门禁见 + [DIFF-4x](rules-system-runtime.md#diff-4x-symmmem-ulysses-a2a-只能显式启用并保持-workspace-生命周期闭合)， + 配置投影见 [VOMNI-CFG-1q](../configuration/rules-diffusion-parallel-transport.md#vomni-cfg-1q-symmmem-ulysses-transport-开关必须完整投影且默认关闭)。^[PR #6340] - 并行度 × 设备容量的启动验收硬规则在 [Model Executor 规则](../model-executor/rules.md)。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index a87ae08c9..391a092e4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -1,10 +1,10 @@ --- title: "Diffusion paged cache 与系统运行时规则" created: 2026-09-03 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/distributed/test_a2a_permute.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] confidence: high --- @@ -80,6 +80,13 @@ confidence: high - 验收：32Q/8KV、异构 prefix、单 token/单 head、非对称等维 LSE 和 batch compaction 精确 对照 dense/reference 输出。 ^[PR #5543] ^[PR #6102] +## DIFF-4x — SymmMem Ulysses A2A 只能显式启用并保持 workspace 生命周期闭合 + +- 触发：修改 `ulysses_a2a_permute`、strict Ulysses all-to-all、NCCL SymmetricMemory JIT extension、CUDA graph capture 或 diffusion worker shutdown。 +- 强制：默认关闭；只在 `ulysses_degree > 1`、strict Ulysses 的正向 `(scatter,gather)=(2,1)` 与反向 `(1,2)` layout 启用，并在 worker/model 初始化时 JIT build/capability failure 直接 fail-at-init。advanced-UAA 保持其 `_ulysses_all_to_all_any_*` path；AllGather 不选择 Ulysses；strict Ulysses+Ring 的 Ulysses leg 仍可使用该 path。每个 `(device, process-group)` 只有一个单 CUDA stream 的 grow-only byte workspace；首次分配及增长必须是 collective、在增长前同步，容量内按 typed view 复用；graph capture 中禁止增长，须先 warm up 最大 shape；worker shutdown 同步并释放 workspace。 +- 禁止：把 flag 当作默认 transport、静默 fallback；按历史 shape/dtype 无界缓存 allocation；跨 stream 重用 staging buffer；capture 中隐式分配/扩容；把 CPU/mock workspace 测试或 PR benchmark 写成数学/输出 parity、普遍速度或跨硬件收益。 +- 验收：覆盖 CLI/deploy/default-stage 的显式 flag 透传与默认 false、eligible strict path 的 init build、UAA/AllGather/noneligible layout 不调用该 path、strict Ulysses+Ring 的 eligible Ulysses leg、peak workspace 复用/collective growth、capture growth fail、single-stream reject 与 shutdown release；真实多 rank hardware 仍须对 regular Ulysses 做 Q/K/V 与 reverse-output parity，再单独记录精确 shape/stack/预热后的性能。^[PR #6340] + ## DIFF-4g — Ring GQA 只在通信后扩展，Hybrid Ulysses 不把 padding 当真 head - 触发：修改 Ring SDPA、GQA/MQA head 数、LSE accumulation、Ulysses all-to-all 或 @@ -183,4 +190,3 @@ confidence: high - 强制：Scheduler 只拥有 logical block allocation、generation 和 `DiffusionKVMetadata`；每个 Worker rank 消费自己的 `KVCacheConfig`，拥有物理 KV tensors、native `BlockTables` 和 request/sequence/context rows。安装 metadata 前必须校验 group/count/range/capacity，并以 staged append、apply 和失败 rollback 原子发布；重复 generation 幂等，冲突或过期快照拒绝，BlockTables 变更使已准备的 attention batch 失效，终止时先清 Worker rows 而不释放 Scheduler blocks。 - 禁止：Worker 自行分配或释放 Scheduler-owned logical blocks；把部分 row 安装、native append/apply 异常或 stale snapshot 当作成功；从 padding 猜异构 prefix 的逻辑布局；在清理或 wake refresh 后继续复用旧的 native metadata/buffer view。 - 验收：CPU/mock 覆盖 rank-local config、请求/上下文 row mapping、重复/过期/冲突 generation、非法 block 与容量、append/apply rollback、幂等清理和 wake refresh；GPU 另验证非连续 BlockTables、GQA、异构 prefix 与 piecewise attention。该 PR 不证明跨 stage KV transport、W2 prefix-cache 或真实 HunyuanImage3 E2E。^[PR #6102] - diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 1026e18e4..95bf94536 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -32,7 +32,7 @@ confidence: high | loader-owned offload plan、backend enable/prefetch failure、fresh canonical retry | `checkpoint-distributed`：`DIFF-12a` | `offloader/startup.py` → `offloader/__init__.py::enable_offload_backend` → runner | | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | Wan VAE spatial shard、gather/trim/reshard、empty tail、attention extent | `distributed-vae`：`DIFF-3b` | `distributed/autoencoders/wan_spatial_shard.py` → patched decoder attention/conv → gather final frame | -| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、FlashInfer quant、masked cross-attention、预算与 admission | `system-runtime`：`DIFF-4a`–`4j`, `DIFF-4v` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | +| paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、SymmMem A2A、FlashInfer quant、masked cross-attention、预算与 admission | `system-runtime`：`DIFF-4a`–`4j`, `DIFF-4v`, `DIFF-4x` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | | worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | | worker title、log prefix、DP/PP/SP/CFG/TP/FS/RP/EP local rank | [worker observability](rules-worker-observability.md)：`DIFF-4w` | `diffusion_worker.py::_setup_diffusion_worker_proc_title_and_log_prefix` → `distributed/parallel_state.py` group getters | | image/diffusion service time、forward time、missing-versus-zero、per-step metric | [`metrics-evidence`](rules-metrics.md)：`DIFF-5a`, `DIFF-5b` | output formatter/sampling metadata → stage stats aggregation → serving Prometheus emission | @@ -46,7 +46,7 @@ confidence: high | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s`, `2y`, `2z` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r`, `2w`, `12a` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o`, `2x` 见 [LoRA](rules-lora.md) | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、能力 metadata、异常与并发 | `DIFF-4a`–`4j`、`4v`（`4a`–`4i` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)，`4v` 见 [attention rules](rules-attention.md)） | +| `system-runtime` | cache/预算、native/backend/platform、attention layout、SymmMem A2A、能力 metadata、异常与并发 | `DIFF-4a`–`4j`、`4v`、`4x`（`4a`–`4i`、`4x` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)，`4v` 见 [attention rules](rules-attention.md)） | | [`metrics-evidence`](rules-metrics.md) | diffusion timing、step normalization、missing-versus-zero、benchmark interpretation | `DIFF-5a`, `DIFF-5b` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 06a91e680..b9cd36ada 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 99a2ae1f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8be601ed` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b1a424e89..a968b5796 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 99a2ae1f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8be601ed` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 7d2783b95..658f6e910 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 99a2ae1f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8be601ed` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 086f86264..3f62e4bb5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 99a2ae1f` 验证存在 +- 源码校验：以上路径均已在 `main @ 8be601ed` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 03c1398da..a5e78f6fd 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 99a2ae1f`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8be601ed`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 21dd4b33 \ - --to 99a2ae1f \ + --from 99a2ae1f \ + --to 8be601ed \ --repo \ --mode report-only ``` From 64fd4f9bcee0eccfa4553cf976ba1773dc44105b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:26:39 +0800 Subject: [PATCH 387/465] knowledge(vllm-omni): sync a6b559cf (PR #6640) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-loader-contract.md | 21 +++++++++++-- .../components/model-executor/rules.md | 3 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/minimax-music3/_index.md | 6 ++-- .../vllm-omni/models/minimax-music3/rules.md | 30 +++++++++++++++++++ 13 files changed, 66 insertions(+), 18 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/minimax-music3/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 896f13d6b..d89355f31 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 99a2ae1fc8637e63a5736a0da68c087caa5f633b + previous_audited_sha: 8be601ed7f2de17b524da47a5f6e383254743e22 audited_ref: main - audited_sha: 8be601ed7f2de17b524da47a5f6e383254743e22 + audited_sha: a6b559cf54ff1c3c8c0a386f3c93fcd8ff1bde41 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index df6359528..2eb2d6f77 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8be601ed7f2de17b524da47a5f6e383254743e22`. The canonical machine baseline +a6b559cf54ff1c3c8c0a386f3c93fcd8ff1bde41`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 76985273c..39260b728 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8be601ed` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a6b559cf` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6b89f42aa..779ee492f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8be601ed` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ a6b559cf` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b9cd36ada..89ffdfb05 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8be601ed` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a6b559cf` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a968b5796..1c95e571c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8be601ed` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a6b559cf` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md index eed7917fe..d05f1955f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md @@ -1,10 +1,10 @@ --- title: "loader 合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py", "PR #6138"] +sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py", "PR #6138", "PR #6640", vllm_omni/engine/stage_init_utils.py, tests/engine/test_async_omni_engine_stage_init.py, tests/engine/test_stage_engine_args.py] confidence: high --- @@ -78,3 +78,20 @@ confidence: high - 禁止：在共享 helper 内按 model type 分支选择校验、恢复可选的模型专属尺寸参数，或让未通过模型校验的 profile 进入 capability 列表和 prompt 路径。 - 验收：分别用 Qwen3-TTS 与 VoxCPM2 的合法、错误尺寸、错误元数据和错误 tensor profile 测试回调结果；确认 adapter 载入与模型运行时载入只保留通过校验的 profile，并覆盖 validator 错误时的跳过行为。^[PR #6138] +## EXEC-2h — stage subdir 必须从完整且同 revision 的 snapshot 解析 + +- 触发：stage `model_subdir`/`tokenizer_subdir`、Hub cache repair、`revision`/ + `tokenizer_revision`、`download_dir` 或 Run:AI object URI model reference。 +- 强制：本地目录缺少声明的 subdir 时立即失败；Hub candidate 只有该 stage 所需的 config、loadable + weight artifact 与 index 引用的全部 shards 完整时，才可改写为本地 subdir；partial cache 必须按同一 + repo/revision/download_dir 请求精确 subset 并复验。仅当 `tokenizer_subdir is not None` 且 + `tokenizer_revision != revision` 时单独解析 tokenizer snapshot；空 tokenizer subdir 表示 snapshot + root，仍须验证 vocabulary artifact。`is_runai_obj_uri` 命中时保留 URI 与 joined subpath，跳过 + local/HF snapshot 推断，交给 upstream streamer。 +- 禁止：以 `isdir`、config/index 文件存在或单 shard 证明 stage 完整；partial snapshot 静默变成本地 + model path；把显式 tokenizer revision 改回 model revision；把 Run:AI URI 当作本地/HF cache，或把 + generic stage resolver 描述成会补齐模型专有的全部 composite root components。 +- 验收：覆盖 cold/partial/warm offline stage snapshot、index 缺 referenced shard、root tokenizer、 + split tokenizer revision、download failure fail-closed 和 Run:AI URI passthrough；模型专有 root + completeness 另由 owner 验证。unit fixture 不证明真实 checkpoint load、音质、GPU E2E 或跨平台支持。 + ^[PR #6640] diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 9b88ea912..86afe5712 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -23,6 +23,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | | cross-stage embedding width、pre-projection buffer | `bridge-batch`：`EXEC-1d` | stage HF config → `get_inputs_embeds_size()` → `GPUARModelRunner.inputs_embeds` → model projection | | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | +| stage `model_subdir`/`tokenizer_subdir`、partial snapshot、revision/cache repair | `loader-contract`：`EXEC-2h` | `engine/stage_init_utils.py` → stage engine args → model-specific root resolver | | fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | H3 text encoder fused owner；其他模型先确认目标 main 是否已有 fused parameter | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | upstream runner/output API drift、`num_nans`、sample-budget 或 quant loader helper moved | `async-output` + output-contract | live upstream runner/output dataclasses → `gpu_{ar,generation}_model_runner.py` / `outputs/output_processor.py` → diffusion quant configs；moved symbol 必须用目标 release 的 canonical import，而非保留旧 fallback ^[PR #6606] | @@ -38,7 +39,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | | `connector-capability` | full-payload/connector stage 的 worker runner validation | `EXEC-3c` | | `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`–`EXEC-1i`，见 [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | -| `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装 | `EXEC-2a`–`EXEC-2c`，见 [loader 合同](rules-loader-contract.md) | +| `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装、stage snapshot/subdir | `EXEC-2a`–`EXEC-2h`，见 [loader 合同](rules-loader-contract.md) | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a`, `EXEC-5b` | | `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | | `image-task-envelope` | shared image/video task example、video output consumer 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6d`，见 [image task envelope 合同](rules-image-task-envelope.md) | diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 658f6e910..ff796924d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8be601ed` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a6b559cf` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3f62e4bb5..1a957e7c7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8be601ed` 验证存在 +- 源码校验：以上路径均已在 `main @ a6b559cf` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a5e78f6fd..313357817 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8be601ed`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a6b559cf`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 99a2ae1f \ - --to 8be601ed \ + --from 8be601ed \ + --to a6b559cf \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-music3/_index.md b/knowledge/repos/vllm-omni/models/minimax-music3/_index.md index 7b059e4a7..bf28a1c96 100644 --- a/knowledge/repos/vllm-omni/models/minimax-music3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-music3/_index.md @@ -1,10 +1,10 @@ --- title: "MiniMax Music3" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, serving] -sources: [vllm_omni/model_executor/models/minimax_music3/, vllm_omni/deploy/minimax_music3.yaml, vllm_omni/deploy/minimax_music3_2gpu.yaml, vllm_omni/entrypoints/openai/tts_adapters/minimax_music3.py, vllm_omni/model_executor/stage_input_processors/minimax_music3.py, vllm_omni/transformers_utils/configs/minimax_music3.py] +sources: ["PR #6640", vllm_omni/model_executor/models/minimax_music3/, vllm_omni/deploy/minimax_music3.yaml, vllm_omni/deploy/minimax_music3_2gpu.yaml, vllm_omni/entrypoints/openai/tts_adapters/minimax_music3.py, vllm_omni/model_executor/stage_input_processors/minimax_music3.py, vllm_omni/transformers_utils/configs/minimax_music3.py] confidence: high --- @@ -50,4 +50,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | +| composite checkpoint、stage subdir、snapshot revision/root | [rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/minimax-music3/rules.md b/knowledge/repos/vllm-omni/models/minimax-music3/rules.md new file mode 100644 index 000000000..1d5c7a1e3 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/minimax-music3/rules.md @@ -0,0 +1,30 @@ +--- +title: "MiniMax Music3 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, model-executor] +sources: ["PR #6640", vllm_omni/model_executor/models/minimax_music3/acoustic.py, vllm_omni/model_executor/models/minimax_music3/weights.py, tests/model_executor/models/test_minimax_music3_repo_root.py] +confidence: high +--- + +# MiniMax Music3 规则 + +## MM3-1a — composite checkpoint 的直接组件必须与 AR snapshot 同 revision 且完整 + +- 触发：MiniMax-Music3 从 Hub/cache/root/`language_model/` 初始化 AR 或 acoustic stage、恢复 repo + identity/revision，或加载 root checkpoint component。 +- 强制：`language_model/` 由 vLLM stage loader 所有；`rvq_depth_decoder/`、 + `condition_encoder/`、`transformer/` 与 `vocoder/` 是从同一 composite repo root 直接 strict-load 的 + 必需组件。root markers 只能用于发现 candidate，不能代替四组件 completeness；进入任一直接加载前， + 每个组件都必须有可加载的单文件 safetensors，或完整的 numeric `-N-of-M` shard run。若从 cached + `language_model/` 进入，必须恢复 Hub repo identity 与 `snapshots/` revision，并将该 revision/ + `download_dir` 传给 cache repair，使直接组件与 AR snapshot 对齐。非标准 shard 名只有在 + `load_component_state` 本身可加载时才接受；component `state_dict` 保持 `strict=True`。 +- 禁止：stage 0 只下载 language model/tokenizer 后就把 root components 当作 ready；以 marker、目录、 + index 或单个 numeric shard 证明完整；遗漏不在 marker set 中的 `condition_encoder`；从 cached + subdir 回退到 `main`，或把 model-specific root recovery 误写成独立 tokenizer revision 传播。 +- 验收：覆盖 Hub ID、repo root 与 cached `language_model/` 三种入口，cold/partial/complete offline cache、 + 缺 `condition_encoder`、缺一个 numeric indexed shard、snapshot revision recovery、custom loadable + shard naming 与 strict-load failure；真实 checkpoint/audio E2E、音质、吞吐和跨平台仍需独立证据。 + ^[PR #6640] From a3060500d3567e3d03cb7288b76df2a3b8d23033 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:30:11 +0800 Subject: [PATCH 388/465] knowledge(vllm-omni): sync c4192568 (PR #6422) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../components/configuration/rules.md | 6 ++-- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-bridge-batch.md | 7 +++-- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/higgs-audio/_index.md | 7 +++-- .../models/higgs-audio/architecture.md | 11 ++++---- .../vllm-omni/models/higgs-audio/rules.md | 28 +++++++++++++++++++ 14 files changed, 58 insertions(+), 25 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/higgs-audio/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d89355f31..272052cc2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8be601ed7f2de17b524da47a5f6e383254743e22 + previous_audited_sha: a6b559cf54ff1c3c8c0a386f3c93fcd8ff1bde41 audited_ref: main - audited_sha: a6b559cf54ff1c3c8c0a386f3c93fcd8ff1bde41 + audited_sha: c41925689d11e99ff20f0cd98a6140d795686790 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2eb2d6f77..56c9bf9f5 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -a6b559cf54ff1c3c8c0a386f3c93fcd8ff1bde41`. The canonical machine baseline +c41925689d11e99ff20f0cd98a6140d795686790`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 39260b728..ff5c43fa3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a6b559cf` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c4192568` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index c3ccaed6c..6db24e6de 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-05 type: rule tags: [vllm-omni, components, config] -sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820", "PR #6619", "PR #6680"] +sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR #5073", "PR #5671", "PR #5678", "zuiho-kai/claude-workflow-starter@c217fc6", vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/config/omni_config.py, vllm_omni/config/composable_parallel/, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_arg_utils.py, tests/engine/test_stage_engine_args.py, "PR #4795", "PR #5842", "PR #6082", "PR #6156", "PR #5741", "PR #6068", "PR #4765", "PR #5666", "PR #4222", "PR #5604", "PR #6293", "PR #6094", "vllm_omni/diffusion/data.py", "PR #6050", "PR #6322", "vllm_omni/config/pipeline_registry.py", "vllm_omni/diffusion/models/pi0_pipeline_config.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "PR #5048", "PR #6458", "PR #6102", "PR #6308", "PR #6182", "PR #4820", "PR #6619", "PR #6680", "PR #6422", vllm_omni/deploy/higgs_multimodal_qwen3.yaml] --- # vLLM-Omni 配置开发门禁 @@ -103,9 +103,9 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR ### VOMNI-CFG-1k — 模型专用 attention 子配置必须贯穿 deploy 与 typed projection - 触发：模型 deploy profile 需要通过 vLLM 顶层 attention backend 下的 `attention_config` 控制具体 decode kernel，或新增该嵌套字段并要求 structured、legacy、typed 路径保持一致。 -- 强制：将 `attention_config` 纳入 `_ModelEngineOverrides` 与 `OmniStageModelConfig`，确保它从 deploy profile 投影到最终 stage engine args。Higgs 的三个 profile 必须在 Stage 0 同时设置 `attention_backend: FLASHINFER` 与 `attention_config.use_trtllm_attention: false`，Stage 1 保持不变。 +- 强制：将 `attention_config` 纳入 `_ModelEngineOverrides` 与 `OmniStageModelConfig`，确保它从 deploy profile 投影到最终 stage engine args。Higgs CUDA profiles 的 Stage 0 保持 `FLASHINFER` 与 `use_trtllm_attention: false`；ROCm overlay 只将 Higgs V3 Stage 0 设为 `TRITON_ATTN`，Stage 1 保持不变。 - 禁止：把 `FLASHINFER` 视为原生 FlashInfer decode 的保证；依据八个 codebook 推断 attention 为 `q_len=8`；只修改一个 Higgs profile；或把 SM90/XQA capability 通过误认为 BF16 工作负载的性能结论。 -- 验收：配置 schema、structured/legacy projection 测试均能读回 `{"use_trtllm_attention": false}`；解析三个 Higgs YAML 并断言 Stage 0 的 backend 与 nested pin 一致，最终日志为 `flashinfer-native`，并用同一栈 A/B 验证 TTFP/RTF 回到原生 FlashInfer 的范围。 ^[PR #6068] +- 验收：配置 schema、structured/legacy projection 测试均能读回 `{"use_trtllm_attention": false}`；解析 Higgs CUDA profiles 及 ROCm V3 overlay，断言仅相应 Stage 0 backend 改动。ROCm collection/test evidence 不构成其它 stage、硬件、音质或性能声明。 ^[PR #6068] ^[PR #6422] ### VOMNI-CFG-1l — upstream 配置复用必须保持 structured 边界与生命周期安全 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 779ee492f..8cc0af33c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a6b559cf` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ c4192568` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 89ffdfb05..7b875101e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a6b559cf` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c4192568` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1c95e571c..6355a4240 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a6b559cf` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c4192568` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md index fa5006623..660a92c10 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-bridge-batch.md @@ -1,10 +1,10 @@ --- title: "跨 stage bridge 与 batch 合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, "PR #4765", "PR #5666", "PR #5491", "PR #6186", "PR #5452", "vllm_omni/worker/output/payload_build.py", "PR #6406"] +sources: ["PR #3422", "PR #3642", "PR #4795", "PR #5073", "PR #5074", "PR #5310", "PR #5792", "PR #5842", "PR #5957", "PR #5976", "PR #6422", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/core/sched/output.py, vllm_omni/utils/mm_outputs.py, tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py, "PR #4765", "PR #5666", "PR #5491", "PR #6186", "PR #5452", "vllm_omni/worker/output/payload_build.py", "PR #6406"] confidence: high --- @@ -45,6 +45,9 @@ Direct 代码快速入口；loader 与 checkpoint 合同留在该页的 `EXEC-2x `batch*vocab` 临时量；global RNG 状态跨 yield 泄漏到兄弟请求。 - 验收：同 seed 同输出、异 seed 不同输出，batch reorder/compaction 后逐请求结果稳定；全局 RNG 前后相同，计数器证明目标分支实际消费请求参数。 ^[PR #3422] ^[PR #5074] ^[PR #5792] + Higgs V3 将每请求的八个 codebook rows 对齐扩展至 `SamplingMetadata.generators`，并把 GPU decode + delay state 以 request ID（非临时 batch row）保存；finished/replacement 同 ID 时缺失 pool entry 必须 + 触发重同步。^[PR #6422] ## EXEC-1d — cross-stage embedding buffer 必须按 ingress width 分配 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ff796924d..1fc0826ae 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a6b559cf` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c4192568` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1a957e7c7..c62c14d04 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a6b559cf` 验证存在 +- 源码校验：以上路径均已在 `main @ c4192568` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 313357817..dd50b77f5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a6b559cf`（2026-08-29）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c4192568`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 8be601ed \ - --to a6b559cf \ + --from a6b559cf \ + --to c4192568 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/_index.md b/knowledge/repos/vllm-omni/models/higgs-audio/_index.md index a99ef53d9..0bda92c9e 100644 --- a/knowledge/repos/vllm-omni/models/higgs-audio/_index.md +++ b/knowledge/repos/vllm-omni/models/higgs-audio/_index.md @@ -1,15 +1,15 @@ --- title: "Higgs-Audio（V2/V3 双谱系 TTS;higgs_multimodal_qwen3 即 V3）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/higgs_audio_v2/, vllm_omni/model_executor/models/higgs_audio_v3/, vllm_omni/transformers_utils/configs/higgs_audio_v3.py] +sources: ["PR #6422", vllm_omni/model_executor/models/higgs_audio_v2/, vllm_omni/model_executor/models/higgs_audio_v3/, vllm_omni/transformers_utils/configs/higgs_audio_v3.py, tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py, tests/e2e/online_serving/test_higgs_audio_v3.py] --- # Higgs-Audio -以下事实在 `main @ 5d44868e` 复核。 +以下事实在 `main @ c4192568` 复核。 ## 名称与范围（先解开命名结） @@ -53,6 +53,7 @@ sources: [vllm_omni/model_executor/models/higgs_audio_v2/, vllm_omni/model_execu | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | 谱系差异表、delay-pattern 生命周期、流式窗口数学 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | +| V3 request RNG、decode state 与 transcript test oracle | [rules](rules.md) | `HIGGS-1a` | ## 配置与 checkpoint 差异（v2 vs v3 一览） diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md b/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md index 5035cb7d8..c3c5e35a8 100644 --- a/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md +++ b/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md @@ -1,10 +1,10 @@ --- title: "Higgs-Audio 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/model_executor/stage_input_processors/higgs_audio_v2.py, vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py, vllm_omni/deploy/README_higgs_audio_v3.md] +sources: ["PR #6422", vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/model_executor/stage_input_processors/higgs_audio_v2.py, vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py, vllm_omni/deploy/README_higgs_audio_v3.md] --- # Higgs-Audio 架构 @@ -38,9 +38,10 @@ sources: [vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.p `<|tts|>/<|ref_text|>/<|ref_audio|>/<|text|>/<|audio|>` 模板,参考音频用 `-100` 占位符在 prefill 换成 delay 编码后的融合嵌入。 - v3 config 从 checkpoint tokenizer 解析特殊 token 与 eos（缺失即 raise）。 -- 采样注意（v2 YAML 注释记载,v3 未单独佐证）：greedy 会塌缩 codebook - argmax——v2 必须采样（stage-0 参数 temp 1.0/top_p 0.95/top_k 50/seed 42）; - 两谱系 talker 都无 `torch.Generator` 用法,确定性只靠 seed。 +- 采样注意：v2 YAML 注释记载 greedy 会塌缩 codebook argmax，因此 v2 profile 启用 sampling； + #6422 不改变也不重新证明 v2。V3 则显式消费 vLLM 的 per-request `torch.Generator`，将一个 + request 对齐到八个 contiguous codebook rows，并以 request ID 保存/恢复 decode-delay state； + 具体 reorder、finish 与 slot-reuse 合同见 [HIGGS-1a](rules.md)。 ## 从输入到输出的主要流程 diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/rules.md b/knowledge/repos/vllm-omni/models/higgs-audio/rules.md new file mode 100644 index 000000000..4198da583 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/higgs-audio/rules.md @@ -0,0 +1,28 @@ +--- +title: "Higgs-Audio 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, model-executor] +sources: ["PR #6422", vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/worker/gpu_model_runner.py, tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py, tests/e2e/online_serving/test_higgs_audio_v3.py] +confidence: high +--- + +# Higgs-Audio 规则 + +## HIGGS-1a — V3 codec sampling 与 decode state 必须按 request 所有 + +- 触发：Higgs-Audio V3 seeded sampling、八 codebook flatten/compaction、GPU decode delay 或 replacement request。 +- 强制：每个 logical request 的八个 contiguous codebook rows 必须在一次 multinomial 中使用该 + request row 的 `SamplingMetadata.generators[req_idx]`；generator 由 vLLM request lifecycle 提供， + talker 不按 request ID 另建 RNG。`has_codes`、last codes、delay/EOC countdown 与 done state 必须由 + request ID 映射到 pool row；reorder/condense 前保存旧 row、再恢复新顺序。finished ID 先释放映射， + slot 复用时重新初始化；即使 `previous_req_ids == req_ids`，当前 ID 缺 pool entry 也必须重同步。 +- 禁止：共享/global RNG、把一个 generator 按八行独立重采样、按临时 batch row 保存 decode state， + 或把 finished callback 描述成会清零 tensor（fresh allocation 才初始化）。`transcript_expected_text` + 只是在 control-token request 与 spoken reference 不同的测试 helper 中覆盖 assertion，默认取 + `input`；它不是 OpenAI request 字段或 runtime 模型语义。 +- 验收：覆盖同 seed 单独/带邻居 batch、异 seed、reorder/condense 连续性、finish、distinct-ID slot + reuse 与 same-ID replacement，并验证 runner 把 live `input_batch.req_ids` 传入 decode metadata。 + ROCm 配置测试只证明 V3 Stage 0 的 `TRITON_ATTN` overlay/collection；作者报告的单次 MI300X case + 不构成通用确定性、音质、吞吐或跨平台支持。^[PR #6422] From 7e87e7564f6d724c5fbaea14016f8b476e05979f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:33:14 +0800 Subject: [PATCH 389/465] knowledge(vllm-omni): sync 1d75e63f (PR #6174) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 11 ++++++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../repos/vllm-omni/models/dots-tts/_index.md | 10 +++++-- .../repos/vllm-omni/models/dots-tts/rules.md | 30 +++++++++++++++++++ 12 files changed, 60 insertions(+), 15 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/dots-tts/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 272052cc2..41b80b1b7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a6b559cf54ff1c3c8c0a386f3c93fcd8ff1bde41 + previous_audited_sha: c41925689d11e99ff20f0cd98a6140d795686790 audited_ref: main - audited_sha: c41925689d11e99ff20f0cd98a6140d795686790 + audited_sha: 1d75e63f7f7c0fcde83594a90876e20415c8bc52 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 56c9bf9f5..1756132ee 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c41925689d11e99ff20f0cd98a6140d795686790`. The canonical machine baseline +1d75e63f7f7c0fcde83594a90876e20415c8bc52`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 616fe49c6..f66052c27 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464", "PR #6730", "PR #6745", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742", "PR #6650", pyproject.toml, tests/helpers/mark.py, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464", "PR #6730", "PR #6745", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742", "PR #6650", pyproject.toml, tests/helpers/mark.py, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, "PR #6174", .buildkite/cuda/test-weekly.yml, tests/e2e/offline_inference/test_dots_tts_expansion.py] --- # vLLM-Omni CI @@ -26,6 +26,15 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie behavior; it does not itself change runtime, offline-inference, online-serving, or transport implementation. ^[PR #5464] ^[PR #6730] ^[PR #6745] +## dots.tts weekly L4 collection boundary + +- Weekly TTS L4 selects `slow and L4 and tts` with `--run-level full_model`; the three dots.tts + offline cases carry `slow`, `tts` and one-card L4 hardware markers, so collection returns three. + The distinct nightly selector `full_model and L4 and tts` returns zero because these cases do not + carry the `full_model` tier marker. PR #6174 reports collection only and explicitly did not run the + tests on L4; this records OMNI-CI-1a/1c wiring, not a hardware pass or runtime/model validation. + ^[PR #6174] + ## LTX-2 FP8 quality-gate recipe boundary - The H100 full-model `fp8_ltx2` gate compares Lightricks/LTX-2 online FP8 with BF16 using diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ff5c43fa3..3f8d6477c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c4192568` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1d75e63f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8cc0af33c..9ae09d548 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c4192568` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 1d75e63f` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7b875101e..36ebca980 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c4192568` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1d75e63f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6355a4240..1740f324f 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c4192568` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1d75e63f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1fc0826ae..5028c7d3a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c4192568` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1d75e63f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c62c14d04..f5f87f548 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c4192568` 验证存在 +- 源码校验：以上路径均已在 `main @ 1d75e63f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index dd50b77f5..ebf85c288 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c4192568`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1d75e63f`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from a6b559cf \ - --to c4192568 \ + --from c4192568 \ + --to 1d75e63f \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/dots-tts/_index.md b/knowledge/repos/vllm-omni/models/dots-tts/_index.md index cb0f061d7..9d7a16069 100644 --- a/knowledge/repos/vllm-omni/models/dots-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/dots-tts/_index.md @@ -1,10 +1,10 @@ --- title: "dots.tts" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, serving] -sources: [vllm_omni/model_executor/models/dots_tts/, vllm_omni/deploy/dots_tts.yaml, vllm_omni/transformers_utils/configs/dots_tts.py] +sources: ["PR #6174", vllm_omni/model_executor/models/dots_tts/, vllm_omni/deploy/dots_tts.yaml, vllm_omni/transformers_utils/configs/dots_tts.py, tests/e2e/offline_inference/test_dots_tts_expansion.py] confidence: high --- @@ -37,6 +37,12 @@ confidence: high 只查 dots.tts 专有的行为、常量、注册入口和验证合同。 +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| weekly offline E2E scaffold、prompt/queue/noise oracle | [rules](rules.md) | + ## 不放什么 上面列出的共享模块的执行、调度、加载或 serving 合同属于 diff --git a/knowledge/repos/vllm-omni/models/dots-tts/rules.md b/knowledge/repos/vllm-omni/models/dots-tts/rules.md new file mode 100644 index 000000000..ac50a345b --- /dev/null +++ b/knowledge/repos/vllm-omni/models/dots-tts/rules.md @@ -0,0 +1,30 @@ +--- +title: "dots.tts 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, ci] +sources: ["PR #6174", tests/e2e/offline_inference/test_dots_tts_expansion.py] +confidence: high +--- + +# dots.tts 规则 + +## DOTS-1a — weekly E2E scaffold 必须保留 prompt、队列与 request-noise oracle + +- 触发：修改 dots.tts offline E2E、prompt helper、`compute_logits` result queue 或 flow-matching noise test。 +- 强制：以显式 remote-code tokenizer 调 `build_dots_tts_prompt`，构造 + `[文本][文本对应语音]<|audio_gen_start|>` 的真实 Qwen2 token IDs。mixed case 的五个请求 + （一长四短）必须按位置维护 `_results_queue -> logits[i]`：每个 step 的 prefill row 也各贡献一个 + entry，不能压缩后错配。测试引用的既有 flow-matching noise 由 + `blake2b(seed:request_key:noise_step)` 派生 request-local generator；音频 oracle 只检查 finite、 + non-silent 和宽松时长。 +- 禁止：用 bare prompt string 代替真实 prefill scaffold；跳过 prefill queue slot；用本 PR 声称新增 + runtime queue/RNG 修复；把 selector collection、non-silence 或相邻两次一致写成可懂度、speaker + similarity、并发确定性、bitwise parity、性能或真实 L4 成功证据。 +- 验收：zero-shot 输出为 48 kHz、finite/non-silent 且时长 `(0.5,30)` 秒，mixed 五项各在 + `(0.1,30)`；sequential singleton 同 prompt 运行两次，shape 相同且 + `torch.allclose(atol=1e-4)`（默认 nonzero `rtol` 仍生效）。三例都带 `slow`、`tts` 和一张 L4 + hardware marker；weekly `-m "slow and L4 and tts"` 收集 3，nightly + `-m "full_model and L4 and tts"` 收集 0。PR 未在 L4 实跑，真实 weekly run 才能证明硬件结果。 + ^[PR #6174] From 3e4eff69ce40825d158cb0479e89da4468635f7d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:38:46 +0800 Subject: [PATCH 390/465] knowledge(vllm-omni): sync 18f01bef (PR #6674) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 6 +++- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-platform-backends.md | 30 +++++++++++++++++-- .../components/model-executor/rules.md | 5 ++-- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 12 files changed, 47 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 41b80b1b7..956d9e16e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c41925689d11e99ff20f0cd98a6140d795686790 + previous_audited_sha: 1d75e63f7f7c0fcde83594a90876e20415c8bc52 audited_ref: main - audited_sha: 1d75e63f7f7c0fcde83594a90876e20415c8bc52 + audited_sha: 18f01befa76865ab8ab01ddba72aad76eaf18ff6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1756132ee..954f9079f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1d75e63f7f7c0fcde83594a90876e20415c8bc52`. The canonical machine baseline +18f01befa76865ab8ab01ddba72aad76eaf18ff6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index ddcbacdcc..353fde9b7 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-05 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3] confidence: high --- @@ -186,6 +186,10 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload `v0.28.0`；CI Dockerfile 在官方同版本 image 出现后移除临时 wheel reinstall，而不是把旧 0.27 image 上的 ABI repair 带入 release。文档安装 pin 也必须同步；非 release SHA 才恢复 明确的 wheel/ABI block。^[PR #6606] +- NPU v0.28 对齐要求 `Dockerfile.npu`、`.npu.a3`、`.npu.ci`、`.npu.ci.a3` 四个既有 image + 一起继承 matching `vllm-ascend:v0.28.0` 并显式设置 `VLLM_TARGET_DEVICE=npu`，同时保留 spawn + multiprocessing。该四-image 合同不能从 CI image 成功外推为 production、全模型或性能兼容。 + ^[PR #6674] - 验收：image tag 与目标 release 一致；startup 通过共享内存 preflight；目标 lane 证明实际 grader device；每个残余失败独立记录，尤其数值/质量回归不能由 API 兼容测试替代。^[PR #5976] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 3f8d6477c..34e885621 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1d75e63f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 18f01bef` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9ae09d548..6b65305e7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1d75e63f` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 18f01bef` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 36ebca980..aa27117c6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1d75e63f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 18f01bef` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1740f324f..98d593d7d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1d75e63f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 18f01bef` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index f379931ed..b7f4ac462 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -1,16 +1,16 @@ --- title: "平台后端合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102", "PR #6054", "vllm_omni/platforms/npu/platform.py", tests/platforms/npu/test_diffusion_attn_backend_selector.py] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102", "PR #6054", "vllm_omni/platforms/npu/platform.py", tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6674", vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py] confidence: high --- # 平台后端合同 -`EXEC-10a`、`EXEC-12a`–`EXEC-13a`：NPU 平台 runner 的 runtime mode 与 dummy-run 接口、ROCm 分页注意力的 packed KV varlen 路径，以及 NPU 模型补丁的注册与回归。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 +`EXEC-10a`–`10e`、`EXEC-12a`–`EXEC-13h`：NPU 平台 runner 的 runtime mode 与 dummy-run 接口、ROCm 分页注意力的 packed KV varlen 路径，以及 NPU 模型补丁的注册与回归。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 ## EXEC-10a — NPU 平台 runner 必须保持 runtime mode 与 dummy-run 接口合同 @@ -90,3 +90,27 @@ confidence: high - 强制：显式 `FLASH_ATTN`（包括在 NPU 上回退到本地 FlashAttention 的 `FLASH_ATTN_HUB` / `FLASH_ATTN_3_HUB`）和 `RAINFUSION_ATTN` 在 `mindiesd` 可发现时必须在 backend path 返回前导入它；后者的 dense FlashAttention fallback 会在 `start_step` 前或没有可稀疏化 video segment 的 layer 触达 MindIE-SD。该导入须早于任何会让 CANN 固化 custom-op registry 的 regInfo lookup，使 `mindiesd.env` 能先把 vendor dirs 写入 `ASCEND_CUSTOM_OPP_PATH`。 - 禁止：为 `TORCH_SDPA` 或其他不会到达 MindIE-SD kernel 的显式 backend 导入可选 `mindiesd`；不得把 optional-package 损坏扩散成这些 backend 的选择失败，也不得假定所有 RAINFUSION layer 都走 sparse path。 - 验收：CPU/mock selector test 应分别断言 FLASH、两个 hub FLASH fallback 与 RAINFUSION 触发 eager import，TORCH_SDPA 不触发；同时覆盖 `mindiesd` 缺失时显式 FLASH 仍解析、未选择 backend 时才在可用条件下默认 FLASH，否则回退 SDPA。该测试不运行 NPU/CANN，不证明 custom-op 注册、真实 kernel 或 Ascend 生产可用性。^[PR #6054] +## EXEC-10e — NPU runner 只能 shallow-copy 自己要改的 mutable map + +- 触发：NPU AR/generation runner 在 `speculative_config.use_ngram_gpu()` 下把同一 + `SchedulerOutput` 交给会 trim invalid draft 的继承路径，或升级 upstream NPU runner ownership。 +- 强制：AR 与 generation 两条路径都用 `dataclasses.replace` 浅拷贝外层 `SchedulerOutput`，且只 + `.copy()` 两个会被修改的 map：`num_scheduled_tokens` 与 `scheduled_spec_decode_tokens`；其他 immutable/ + tensor state 继续共享。vLLM-Ascend 0.28 已拥有 CUDAGraph/KV-scale handling 与 SP/flashcomm-v1 output + gather 时，Omni 只保留自身 forwarding/output hooks，不重复 reset、calculate 或 gather。 +- 禁止：原地修改 EngineCore-owned maps；`deepcopy(scheduler_output)`；只修 AR 或 generation 一侧； + 恢复已经上移给 upstream 的 KV-scale/CUDAGraph/SP gather，或把删除重复逻辑写成数值/性能已验证。 +- 验收：ngram enabled 时两 runner 的 inherited mutation 都不改变原始两个 map，copy 仅隔离它们； + disabled control 不额外复制。另以 owner/call-count 测试防止 duplicate KV-scale 与 SP gather。 + PR #6674 未新增这些专项执行测试；CPU/mock 也不证明真实 NPU parity。^[PR #6674] + +## EXEC-13h — NPU profiling 只有 enabled 且 need_timing 才进入 timing path + +- 触发：NPU runner profiling、model execute timing 或 vLLM 0.28 profiler API 对齐。 +- 强制：AR 与 generation 都从同一 `profiling_chunk_config` 读取 gate；scheduler output 的 + `disable_profiling_timing` 先把当前 call 的 `need_timing` 置 false。只有 `enabled && need_timing` + 才启动/同步计时，并且只有同一 gate 成立才导出 `execution_time_ms`；其他调用保持 normal path。 +- 禁止：仅因 profiler/config object 存在或 `need_timing=true` 就计时；profiling disabled 时创建 timing + state；只在 start 处 gate、却无条件同步/导出，或将 gate 本身当作 NPU 性能结果。 +- 验收：AR/generation 都覆盖 enabled+timed、disabled+timed、enabled+not-timed 及 per-output disable， + 断言 start/sync/export 一致。PR #6674 没有这些专项测试，属于后续验收要求。^[PR #6674] diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 86afe5712..353ba2c94 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149", "PR #4322"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149", "PR #4322", "PR #6674"] --- # Model Executor 规则 @@ -27,6 +27,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | H3 text encoder fused owner；其他模型先确认目标 main 是否已有 fused parameter | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | upstream runner/output API drift、`num_nans`、sample-budget 或 quant loader helper moved | `async-output` + output-contract | live upstream runner/output dataclasses → `gpu_{ar,generation}_model_runner.py` / `outputs/output_processor.py` → diffusion quant configs；moved symbol 必须用目标 release 的 canonical import，而非保留旧 fallback ^[PR #6606] | +| NPU ngram speculative decode、`SchedulerOutput` mutable map、profiling timing 或 upstream NPU runner ownership | `platform-backends`：`EXEC-10e`, `EXEC-13h` | `platforms/npu/worker/{npu_ar_model_runner,npu_generation_model_runner,npu_model_runner}.py` → inherited invalid-draft trimming / profiler gate → upstream vLLM-Ascend owner | | Talker-MTP FULL graph、平台 capture 能力、显式 opt-out | `mtp-graph`：`EXEC-4c` | `platforms/interface.py::supports_talker_mtp_graph_capture` → platform override → model `talker_mtp_graph_safe` → `OmniGPUModelRunner._init_talker_mtp` | | `model_extras`、shared T2I/T2V/I2V example、canonical prompt envelope 或 video output consumer | `image-task-envelope`：`EXEC-6a`, `EXEC-6d` | `examples/offline_inference/{text_to_image/text_to_image.py,text_to_video/text_to_video.py,image_to_video/image_to_video.py}` → `model_extras/registry.py::{build_text_to_image_prompt,build_image_to_video_prompt}` → model pipeline validation | | CosyVoice3 non-module TensorRT CFM、stream/event ownership、raw-pointer buffer lifetime 或 context-pool reuse | [CosyVoice3 TensorRT handoff](../../models/cosyvoice3/rules.md)：`COSYVOICE3-1a` | `code2wav_core/cfm.py::ConditionalCFM.forward_estimator` → `flow_estimator_trt.py::TrtContextWrapper` | @@ -46,7 +47,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | | `output-contract` | Omni 输出类型与字段/复制合同 | `EXEC-7a`–`EXEC-7b`，见 [输出类型合同](rules-output-contract.md) | | `runtime-hot-paths` | 采样循环不变量、固定输入缓存、AR 音频侧路、codec 帧账本与 CosyVoice3 typed handoff | `EXEC-8a`, `EXEC-9a`, `EXEC-11a`–`EXEC-11h`，见 [运行时热路径合同](rules-runtime-hot-paths.md) | -| `platform-backends` | NPU runner 接口、ROCm 分页注意力、NPU 模型补丁注册 | `EXEC-10a`, `EXEC-12a`–`EXEC-13a`，见 [平台后端合同](rules-platform-backends.md) | +| `platform-backends` | NPU runner 接口、ROCm 分页注意力、NPU 模型补丁注册 | `EXEC-10a`–`EXEC-10e`, `EXEC-12a`–`EXEC-13h`，见 [平台后端合同](rules-platform-backends.md) | ## 严格配置校验 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 5028c7d3a..61991c1d0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1d75e63f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 18f01bef` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f5f87f548..80b8ecfa0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1d75e63f` 验证存在 +- 源码校验：以上路径均已在 `main @ 18f01bef` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ebf85c288..f5ab123a5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1d75e63f`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 18f01bef`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from c4192568 \ - --to 1d75e63f \ + --from 1d75e63f \ + --to 18f01bef \ --repo \ --mode report-only ``` From 5237c5e081f4f707582003131519048061a8c943 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:43:22 +0800 Subject: [PATCH 391/465] knowledge(vllm-omni): sync f7f48929 (PR #6813) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../configuration/rules-topology-profiles.md | 8 ++++---- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 19 +++++++++++++++++-- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../models/minimax-h3/rules-media.md | 9 +++++---- 12 files changed, 38 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 956d9e16e..8be02a2d9 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1d75e63f7f7c0fcde83594a90876e20415c8bc52 + previous_audited_sha: 18f01befa76865ab8ab01ddba72aad76eaf18ff6 audited_ref: main - audited_sha: 18f01befa76865ab8ab01ddba72aad76eaf18ff6 + audited_sha: f7f48929db8dcd60c8ce538f4057bec5a789057c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 954f9079f..97111ef8a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -18f01befa76865ab8ab01ddba72aad76eaf18ff6`. The canonical machine baseline +f7f48929db8dcd60c8ce538f4057bec5a789057c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 34e885621..9b1845a14 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 18f01bef` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f7f48929` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md index f513748d5..9c40e1dad 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md @@ -1,10 +1,10 @@ --- title: "topology 与部署 profile 合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, config] -sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/deploy/, "PR #6186"] +sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_runtime.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/deploy/, "PR #6186", "PR #6813"] confidence: high --- @@ -80,6 +80,6 @@ confidence: high ## CONF-5i — pipeline 专属 alias 与 hook 必须在通用配置层声明、归一并消费 - 触发：pipeline 需要把全局 CLI 字段改写到特定 stage，或声明 stage-0 prompt transform、按 task 解析 checkpoint 的 hook、显式 inline diffusion。 -- 强制：alias 由 `PipelineConfig.stage_cli_aliases` 唯一声明并在 structured/legacy factory 进入 stage merge 前归一；canonical `stage__` 与 alias 同时出现时，canonical 值优先并告警，alias 出现在其他 stage 必须拒绝。`prompt_transform_func`、`model_path_resolver` 与 `inline_diffusion` 由 `StagePipelineConfig` 投影到唯一 consumer；resolver 必须在构造 backend engine args 前消费并移除。inline 只允许单 replica，且须由 `inline_diffusion=true` 显式 opt in，或沿用已有 `custom_pipeline_args` 所声明的 in-process pipeline owner；其他 stage 默认保持 subprocess isolation。 +- 强制：alias 由 `PipelineConfig.stage_cli_aliases` 唯一声明并在 structured/legacy factory 进入 stage merge 前归一；canonical `stage__` 与 alias 同时出现时，canonical 值优先并告警，alias 出现在其他 stage 必须拒绝。`prompt_transform_func`、`model_path_resolver` 与 `inline_diffusion` 由 `StagePipelineConfig` 投影到唯一 consumer；resolver 必须在构造 backend engine args 前消费并移除。所有 inline path 都要求 `plan.num_replicas == 1`；whole pipeline 只有一个 stage 时默认 inline，多 stage 只允许 `inline_diffusion=true` 或已有 `custom_pipeline_args` in-process owner opt in，其余均 subprocess。 - 禁止：在 CLI、factory 或 stage startup 为模型名硬编码 alias/resolver；把 alias 广播到所有 stage；因 diffusion stage 只有一个 replica 就隐式改变所有多 stage pipeline 的进程隔离。 -- 验收：structured/legacy 两路覆盖 alias-only、canonical-only、相同/冲突值、错误 stage 与 unset control；用 root/partition/task path 证明 resolver 到达 stage model/tokenizer consumer，并用 opt-in 与相邻未 opt-in pipeline 证明 inline 只改变声明的 stage。^[PR #5885] +- 验收：structured/legacy 两路覆盖 alias-only、canonical-only、相同/冲突值、错误 stage 与 unset control；用 root/partition/task path 证明 resolver 到达 stage model/tokenizer consumer，并用 single-stage、multi-stage opt-in/control 与 multi-replica veto 证明 exact inline predicate。PR #6813 只新增 default one/two-stage 参数化 test，其他组合仍须补验收。^[PR #5885] ^[PR #6813] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6b65305e7..e9f2d6106 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 18f01bef` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ f7f48929` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index aa27117c6..e154b85e7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 18f01bef` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f7f48929` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 98d593d7d..674168fb0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 18f01bef` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f7f48929` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 353ba2c94..9e4907f68 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149", "PR #4322", "PR #6674"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149", "PR #4322", "PR #6674", "PR #6813"] --- # Model Executor 规则 @@ -19,7 +19,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | strict stage config、known-fields/projection、service/stage 字段归属、runtime config | `strict-stage-config`：`EXEC-3a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_common_stage_config_kwargs,VllmOmniConfig.from_pipeline_config}` → 真实 startup consumer | | connector-required stage、worker/runner selection、platform worker capability | `connector-capability`：`EXEC-3c` | `engine/arg_utils.py::OmniEngineArgs.__post_init__` → `worker/omni_connector_validation.py::validate_worker_omni_connector` → selected `worker_cls.model_runner_cls` | | runner `_preprocess`、逐请求 metadata、prefill/decode phase、batch preprocess、MTP | `runner-preprocess`：本页“Runner 到模型的预处理合同” | `vllm_omni/worker/gpu_model_runner.py::{OmniGPUModelRunner._maybe_run_batch_preprocess,_preprocess,_build_model_kwargs_extra,_talker_mtp_forward}` → `vllm_omni/model_executor/models/<命中模型>` consumer | -| stage TP/PP/DP、devices、replica、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | +| stage TP/PP/DP、devices、replica、inline/subprocess、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：`EXEC-3d` + 本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | | cross-stage embedding width、pre-projection buffer | `bridge-batch`：`EXEC-1d` | stage HF config → `get_inputs_embeds_size()` → `GPUARModelRunner.inputs_embeds` → model projection | | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | @@ -39,6 +39,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `core` | 每次 model-executor 审查 | `EXEC-1a` | | `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | | `connector-capability` | full-payload/connector stage 的 worker runner validation | `EXEC-3c` | +| `stage-runtime` | stage 数、replica、inline/subprocess 与设备容量 | `EXEC-3d` + 本页 stage 并行度验收 | | `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`–`EXEC-1i`，见 [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | | `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装、stage snapshot/subdir | `EXEC-2a`–`EXEC-2h`，见 [loader 合同](rules-loader-contract.md) | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a`, `EXEC-5b` | @@ -78,6 +79,20 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru - 禁止：按 model architecture hard-code connector 初始化 allowlist；等到 request parked 或 connector method 缺失才失败；把无 connector 的普通 async scheduler stage 误判为 connector-required。 - 验收：覆盖显式兼容/不兼容 worker、platform-resolved worker、无 connector configuration 的 control，以及 GPU、NPU、XPU AR/generation worker 的 runner-class 暴露和 mixin 继承。^[PR #6149] +### EXEC-3d — StageRuntime inline 资格必须按 whole-pipeline stage 数、replica 与 owner 精确判定 + +- 触发：修改 diffusion replica 的 `StageRuntime` inline/subprocess 初始化、pipeline + `inline_diffusion`、`custom_pipeline_args` 或 stage/replica topology。 +- 强制：`use_inline` 只能满足 `plan.num_replicas == 1 and (self._num_stages == 1 or + inline_diffusion or custom_pipeline_args)`；其中 `_num_stages` 是 whole-pipeline stage 数。默认仅 + single-stage/single-replica inline；multi-stage 默认 subprocess，仅显式 `inline_diffusion` 或已有 + custom-pipeline in-process owner 可 opt in，且仍须 single replica。 +- 禁止：把“当前 diffusion stage 是一个”错当 whole pipeline single-stage；仅凭一个 replica inline + split Stage 1；让任一 opt-in 绕过 multi-replica 隔离，或称所有 subprocess/共享内存边界都已消除。 +- 验收：至少覆盖默认 single-stage→inline、two-stage→subprocess、multi-stage 两种 opt-in，以及 + multi-replica 对默认/opt-in 的否决。PR #6813 新增的单元测试只参数化默认 one/two-stage，未覆盖 + flag、custom、multi-replica 或真实 in-process execution；其 L20X benchmark 仅是所列 workload 的作者证据。^[PR #6813] + ### EXEC-4a — full-duplex chunk metadata 必须按 span 隔离且可安全序列化 - 触发：`experimental/fullduplex` 修改 PCM/audio span、force-listen、rollback、 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 61991c1d0..5b8ae8772 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 18f01bef` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f7f48929` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 80b8ecfa0..5f9ad9326 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 18f01bef` 验证存在 +- 源码校验：以上路径均已在 `main @ f7f48929` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f5ab123a5..d62ab5d78 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 18f01bef`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f7f48929`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 1d75e63f \ - --to 18f01bef \ + --from 18f01bef \ + --to f7f48929 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md index 6018a3e47..2bb054f4f 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md @@ -1,10 +1,10 @@ --- title: "MiniMax H3 媒体输入与精度规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5752", "PR #5829", "PR #5885", "PR #5978", "PR #6555", "PR #6688", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/model_executor/models/minimax_h3/reference_video.py, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064"] +sources: ["PR #5752", "PR #5829", "PR #5885", "PR #5978", "PR #6555", "PR #6688", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/model_executor/models/minimax_h3/reference_video.py, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/engine/stage_runtime.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_async_omni_engine_stage_init.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064", "PR #6813"] confidence: high --- @@ -107,8 +107,9 @@ confidence: high - 强制：Ref2VA 的 Stage 0 preparation 要把 serialized prepared-video descriptor 和 artifact owner 交给 original request；Stage 1 reuse the RGB stream for VAE/audio conditions, and terminal/error paths must clean it. This avoids duplicate ffprobe/transcode while preserving raw media ownership at Stage 1. - `inline_diffusion` remains explicit pipeline opt-in; a single replica alone must not move another - multi-stage pipeline's diffusion engine into the orchestrator process. Encoder-free DLO must skip absent + split profile 是 multi-stage，不能仅因一个 replica inline Stage 1；fused/bare single-stage H3 仅在一个 + replica 时默认 inline。显式 `inline_diffusion` 或已有 custom-pipeline owner 可让 single-replica split/custom + 路径 opt in，但任何超过一个 replica 的 plan 都必须 subprocess。Encoder-free DLO must skip absent components, and final-output validation follows declared final video metadata rather than requiring every stage to be diffusion. - 禁止：把 split route 自动发现为 H3 default、把 `text_encoder_tp_size` broadcast 到 Stage 1、用 From aa45bc08aa386eb5b3ef7abed629cb5dd300f42e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:47:32 +0800 Subject: [PATCH 392/465] knowledge(vllm-omni): sync 71748388 (PR #6818) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/benchmark/_index.md | 6 +++--- .../benchmark/guides/omniinteract-realtime.md | 15 +++++++++------ knowledge/repos/vllm-omni/ci/rules.md | 17 ++++++++++++++--- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 12 files changed, 38 insertions(+), 24 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8be02a2d9..2989c66c4 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 18f01befa76865ab8ab01ddba72aad76eaf18ff6 + previous_audited_sha: f7f48929db8dcd60c8ce538f4057bec5a789057c audited_ref: main - audited_sha: f7f48929db8dcd60c8ce538f4057bec5a789057c + audited_sha: 71748388f820a9b71d7b73954b99f3c83593c0a8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 97111ef8a..3d429b8ee 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f7f48929db8dcd60c8ce538f4057bec5a789057c`. The canonical machine baseline +71748388f820a9b71d7b73954b99f3c83593c0a8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/benchmark/_index.md b/knowledge/repos/vllm-omni/benchmark/_index.md index 1f12adf69..3c17b247e 100644 --- a/knowledge/repos/vllm-omni/benchmark/_index.md +++ b/knowledge/repos/vllm-omni/benchmark/_index.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 性能与 Profiling" created: 2026-07-10 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, benchmark] -sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml, "PR #6522", "PR #6634", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/benchmarks/duplex/, vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py, vllm_omni/experimental/fullduplex/client.py] +sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml, "PR #6522", "PR #6634", "PR #6818", vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/patch/test_patch.py, vllm_omni/benchmarks/duplex/, vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py, vllm_omni/experimental/fullduplex/client.py] --- # vLLM-Omni 性能与 Profiling @@ -25,7 +25,7 @@ sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml, "PR #6 | 运行 AR graph、Hunyuan 或 MiniCPM-o Omni-DuplexEval benchmark | [benchmark guides](guides/_index.md) | | 调查 profiling、模型加载和性能验证错误 | [benchmark incidents](incidents/_index.md) | | 查询历史结果 | [results](results/_index.md) | -| 运行/审查 OmniInteract native-duplex serving benchmark | [OmniInteract realtime guide](guides/omniinteract-realtime.md) | +| 运行/审查 OmniInteract native-duplex serving benchmark（client ITL 优先，Stage-0 仅为无正 client ITL 的受限恢复） | [OmniInteract realtime guide](guides/omniinteract-realtime.md) | `benchmarks/tts/model_configs.yaml` 是共享 TTS runner registry；新模型在这里声明 model、deploy、voice/ reference 与 workload，结果保留解析配置、seed、speed、并发和音频属性。IndexTTS 2.5 条目只证明 diff --git a/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md b/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md index c3540b36b..763448d40 100644 --- a/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md +++ b/knowledge/repos/vllm-omni/benchmark/guides/omniinteract-realtime.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: guide tags: [vllm-omni, benchmark] -sources: ["PR #6522", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/experimental/fullduplex/openai/protocol.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, docs/cli/bench/serve.md, vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/entrypoints/cli/benchmark/cli_args.py] +sources: ["PR #6522", "PR #6696", "PR #6818", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/experimental/fullduplex/openai/protocol.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, docs/cli/bench/serve.md, vllm_omni/benchmarks/data_modules/omniinteract_dataset.py, vllm_omni/benchmarks/omniinteract.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/entrypoints/cli/benchmark/cli_args.py] confidence: high --- @@ -51,11 +51,14 @@ production deployment contract. ^[PR #6522] timing; ITL is reported only when all required intervals exist. Missing measurements remain missing rather than becoming zero, so request goodput cannot pair metrics from different responses. -- TPOT 样本必须 finite 且严格大于零：优先使用长度完整的 exact ITL；否则只有各 Stage-0 - `output_token_count` 合计与最终 token 数完全一致、且每段 `tpot_ms` 都是正有限值时，才按 decode - interval 数加权恢复 TPOT。native duplex 的逐段 metrics 是增量，需累加；chat fallback 发出的是 - cumulative snapshot，需按 stage 替换最新值。上述来源都不可用时，仅当 client 观察到 finite - `text_latency > ttft > 0` 才可按总 token 数回退计算，否则 TPOT 保持 unavailable。^[PR #6696] +- TPOT 样本必须 finite 且严格大于零。native duplex 的逐段 Stage-0 metrics 是增量，需累加；只有 + token count 完整时使用 exact ITL，否则全部 segment 都有正有限 `tpot_ms` 时才按 interval 加权； + 最后仅在 client 观察到 `text_latency > ttft > 0` 时按总 token 数回退。^[PR #6696] +- `openai-chat-omni` 使用不同的 source precedence：client receive interval 只要有正有限 sample 就是 + authoritative；只有没有正 client ITL 且最终 token 数大于一，才读取 latest cumulative Stage-0 + snapshot。其正 `num_tokens_out` 必须与最终累计 usage 完全相等；完整 finite/nonnegative exact ITL + 且总和为正优先，否则用正有限 Stage-0 TPOT 按 interval 投影。不得由 server 覆盖正 client timing， + 也不得混合 snapshot 或把 mismatch/zero/NaN 变成样本；无有效来源时保持 unavailable。^[PR #6818] ## Artifact and evidence contract diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 353fde9b7..2afe78695 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-05 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818"] confidence: high --- @@ -216,9 +216,20 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload ## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 - 触发：修改 perf JSON 的 `baseline`、硬件 label/marker、benchmark result schema，或恢复性能阈值断言。 -- 强制：TPOT 只接受 finite positive value；优先 exact ITL，其次 Stage0 weighted fallback，最后仅在 +- 强制：TPOT 只接受 finite positive value；`openai-chat-omni` 的 client receive interval 只要有一个 + finite positive sample 就保持 client timing，不得由 server 覆盖。只有选择 TPOT/ITL percentile 时才请求 + stage metrics，且 client 没有正 interval、最终 `output_tokens > 1` 时才读取 latest cumulative Stage 0。 + Stage-0 `num_tokens_out` 必须是正数且与最终累计 usage completion-token count 精确相等：长度恰为 + `output_tokens - 1` 的 finite/nonnegative ITL 且总和为正时使用 exact ITL；否则只接受 positive finite + Stage-0 TPOT，并按 decode interval 投影 `text_latency`。两者都无效则 unavailable；不得混合来源、 + 使用其他 stage、混合 cumulative snapshots、制造 zero 或接受 NaN。native-duplex 的最后回退仅在 `text_latency > ttft > 0` 时用 latency fallback。native duplex 累加 segment counts，chat cumulative - snapshots 替换；configured mean TPOT baseline 需 positive sample count 和 finite aggregate。JSON 仍非 strict，NaN gap 未闭合。^[PR #6696] + snapshots 替换；configured mean TPOT baseline 需 positive sample count 和 finite aggregate。JSON 仍非 strict，NaN gap 未闭合。^[PR #6696] ^[PR #6818] +- 验收：覆盖 `openai-chat-omni` 只在 TPOT/ITL 被选择时 opt in stage metrics；首个 SSE bundle 全部 + token 时用 count-matched Stage-0 exact ITL 恢复；client 有正 interval 时即使 Stage 0 不同也保持 + client；另需覆盖 count mismatch、缺失/非法 Stage 0、zero-only ITL 与 TPOT-only weighted fallback，且 + 无有效样本不能让 baseline 通过。PR #6818 新增测试只直接覆盖前三项中的 opt-in、bundle recovery 与 + client precedence，不证明其余负向分支或跨平台性能。^[PR #6818] - 强制：baseline 只接受 `{hardware: {metric: threshold}}` 的硬件嵌套结构；顶层 hardware 必须是 `pyproject.toml` 中带 `[hardware-resource]` 标签的 pytest marker，每个 bucket 非空。metric 名可 自定义，值只能是单点 scalar 或与 `request_rate` / `max_concurrency` 顺序严格对齐的 list。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9b1845a14..69d113190 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f7f48929` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 71748388` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e9f2d6106..0fbe23cac 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f7f48929` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 71748388` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e154b85e7..6933c6fd0 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f7f48929` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 71748388` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 674168fb0..2c4fd5b55 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f7f48929` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 71748388` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 5b8ae8772..89619fa4e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f7f48929` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 71748388` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5f9ad9326..ad06fcbc1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f7f48929` 验证存在 +- 源码校验：以上路径均已在 `main @ 71748388` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d62ab5d78..ec89278d8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f7f48929`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 71748388`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 18f01bef \ - --to f7f48929 \ + --from f7f48929 \ + --to 71748388 \ --repo \ --mode report-only ``` From 4cc9609e98debfef2f34ffb373b563b2a0fc8fbb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:52:09 +0800 Subject: [PATCH 393/465] knowledge(vllm-omni): sync 8fe913e4 (PR #6291) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../configuration/rules-topology-profiles.md | 17 +++++++++++++++-- .../vllm-omni/components/configuration/rules.md | 4 ++-- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 29 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2989c66c4..0a98fabd9 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f7f48929db8dcd60c8ce538f4057bec5a789057c + previous_audited_sha: 71748388f820a9b71d7b73954b99f3c83593c0a8 audited_ref: main - audited_sha: 71748388f820a9b71d7b73954b99f3c83593c0a8 + audited_sha: 8fe913e461d45a1b2d3d0f3d5ecfe1d2b3210996 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3d429b8ee..56d5fbef3 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -71748388f820a9b71d7b73954b99f3c83593c0a8`. The canonical machine baseline +8fe913e461d45a1b2d3d0f3d5ecfe1d2b3210996`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 69d113190..e131e1551 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 71748388` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8fe913e4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md index 9c40e1dad..c8908a933 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md @@ -4,13 +4,13 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, components, config] -sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_runtime.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/deploy/, "PR #6186", "PR #6813"] +sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_runtime.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/deploy/, "PR #6186", "PR #6813", "PR #6291", tests/config/test_config_factory.py, tests/config/test_pipeline_registry.py] confidence: high --- # topology 与部署 profile 合同 -`CONF-5a`–`CONF-5i`：冻结 topology 的单一来源、可选辅助 stage 注入、缺少 model_type 的多 stage checkpoint 路由，以及模型专用 deploy profile 的一致传递。触发条件与其余审查组见 [Configuration 规则](rules.md) 的 Direct 代码快速入口。 +`CONF-5a`–`CONF-5j`：冻结 topology 的单一来源、可选辅助 stage 注入、缺少 model_type 的多 stage checkpoint 路由，以及模型专用 deploy profile 的一致传递。触发条件与其余审查组见 [Configuration 规则](rules.md) 的 Direct 代码快速入口。 ## CONF-5a — 冻结 topology 只保留一份，部署开关决定 wiring @@ -83,3 +83,16 @@ confidence: high - 强制：alias 由 `PipelineConfig.stage_cli_aliases` 唯一声明并在 structured/legacy factory 进入 stage merge 前归一；canonical `stage__` 与 alias 同时出现时，canonical 值优先并告警，alias 出现在其他 stage 必须拒绝。`prompt_transform_func`、`model_path_resolver` 与 `inline_diffusion` 由 `StagePipelineConfig` 投影到唯一 consumer；resolver 必须在构造 backend engine args 前消费并移除。所有 inline path 都要求 `plan.num_replicas == 1`；whole pipeline 只有一个 stage 时默认 inline，多 stage 只允许 `inline_diffusion=true` 或已有 `custom_pipeline_args` in-process owner opt in，其余均 subprocess。 - 禁止：在 CLI、factory 或 stage startup 为模型名硬编码 alias/resolver；把 alias 广播到所有 stage；因 diffusion stage 只有一个 replica 就隐式改变所有多 stage pipeline 的进程隔离。 - 验收：structured/legacy 两路覆盖 alias-only、canonical-only、相同/冲突值、错误 stage 与 unset control；用 root/partition/task path 证明 resolver 到达 stage model/tokenizer consumer，并用 single-stage、multi-stage opt-in/control 与 multi-replica veto 证明 exact inline predicate。PR #6813 只新增 default one/two-stage 参数化 test，其他组合仍须补验收。^[PR #5885] ^[PR #6813] + +## CONF-5j — PipelineConfig 终端 stage 必须显式声明且不依赖位置 + +- 触发：修改 `PipelineConfig.validate()`、静态 `OMNI_PIPELINES` registry，或新增/调整 + multi-stage pipeline 的 `final_output` 声明。 +- 强制：保留既有「没有 entry stage」校验，并要求至少一个 stage 显式 + `final_output=True`；terminal 可以处于任意 position。静态 `PipelineConfig` registry + 必须逐项通过该 topology guard。 +- 禁止：从最后一个 stage、stage 数、entry/input、`final_output_type` 推断 terminal；不得把 + 此静态 guard 外推为 cycle 检测、resolver 配置、运行时或 E2E 行为已经验证。 +- 验收：无 terminal 的配置返回错误；terminal-not-last 通过；有效 fixture 显式标记 + terminal；所有静态 `PipelineConfig` 通过。resolver 生成的配置须在 materialize 后另测。 + ^[PR #6291] diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 6db24e6de..693e172c7 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -21,7 +21,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| | strict schema、unknown field、alias、flat→nested、structured/legacy/direct parity、typed projection | `strict-normalization`：`VOMNI-CFG-1a`–`1h` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,strip_parent_engine_args}` → `vllm_omni/config/omni_config.py::{_build_diffusion_config_projection,VllmOmniConfig.from_pipeline_config}` → `vllm_omni/engine/stage_init_utils.py::{build_engine_args_dict,build_engine_args_dict_from_omni_stage_config}` | -| deploy YAML、`base_config`、pipeline/stage overlay、pipeline-owned alias/hook、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a`, [CONF-5i](rules-topology-profiles.md#conf-5i-pipeline-专属-alias-与-hook-必须在通用配置层声明归一并消费) | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,normalize_pipeline_cli_overrides,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::StageConfigFactory.create_from_model` → `vllm_omni/engine/stage_init_utils.py::_resolve_model_path` | +| deploy YAML、`base_config`、pipeline/stage overlay、pipeline-owned alias/hook、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a`, [CONF-5i](rules-topology-profiles.md#conf-5i-pipeline-专属-alias-与-hook-必须在通用配置层声明归一并消费), [CONF-5j](rules-topology-profiles.md#conf-5j-pipelineconfig-终端-stage-必须显式声明且不依赖位置) | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,normalize_pipeline_cli_overrides,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::StageConfigFactory.create_from_model` → `vllm_omni/engine/stage_init_utils.py::_resolve_model_path` | | MiniCPM-o shipping profile、duplex/chat 共服、Talker sampling floor | model-specific [`MCPMO-4e`](../../models/minicpm-o-4-5/rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → `merge_pipeline_deploy`；duplex adapter runtime sampling override | | composable strategy、axis、routing、load balancing、`strategy-config` | `composable-strategy`：`CONF-4a` | `vllm_omni/config/composable_parallel/strategy_loader.py::{parse_strategy_specs,load_strategy_specs}` → `translator.py::translate_strategy_stack` → `apply.py::apply_strategy_specs` → `config_factory.py::{StageConfigFactory._apply_strategy_specs,StageConfigFactory._reconcile_strategy_with_cli}` | | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | @@ -37,7 +37,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | `core` | 每次配置审查 | `VOMNI-CFG-1b`, `VOMNI-CFG-1c` | | `strict-normalization` | schema、alias、unknown field、structured/legacy/direct 路径 | `VOMNI-CFG-1a`, `VOMNI-CFG-1b`, `VOMNI-CFG-1c`, `VOMNI-CFG-1d`, `VOMNI-CFG-1e`, `VOMNI-CFG-1f`, `VOMNI-CFG-1g`, `VOMNI-CFG-1h` | | `deploy-memory` | 显存预算、KV pin、多 stage 共卡 | `CONF-1a`, `CONF-2a` | -| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5i` 见 [topology 与部署 profile](rules-topology-profiles.md)；MiniCPM-o profile 见 `MCPMO-4e` | +| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5j` 见 [topology 与部署 profile](rules-topology-profiles.md)；MiniCPM-o profile 见 `MCPMO-4e` | | `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a`, `CONF-4c`–`4d`，见 [并行拓扑合同](rules-parallel-topology.md) | | `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | | `stage-config-propagation` | pipeline sampling constraints、StageConfig serialization/runtime config | `VOMNI-CFG-1o` | diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0fbe23cac..3e6bb7bbe 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 71748388` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 8fe913e4` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6933c6fd0..6c61ea057 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 71748388` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8fe913e4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2c4fd5b55..c2d87a669 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 71748388` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8fe913e4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 89619fa4e..2569f9e43 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 71748388` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8fe913e4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ad06fcbc1..a28847e94 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 71748388` 验证存在 +- 源码校验：以上路径均已在 `main @ 8fe913e4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ec89278d8..0a93bef4a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 71748388`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8fe913e4`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from f7f48929 \ - --to 71748388 \ + --from 71748388 \ + --to 8fe913e4 \ --repo \ --mode report-only ``` From aa4321df8053c8ae0f733e1f63a017094e4ad27c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 04:54:50 +0800 Subject: [PATCH 394/465] knowledge(vllm-omni): sync 1a9d598d (PR #6776) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/rules.md | 8 ++++---- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0a98fabd9..a546b7b9f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 71748388f820a9b71d7b73954b99f3c83593c0a8 + previous_audited_sha: 8fe913e461d45a1b2d3d0f3d5ecfe1d2b3210996 audited_ref: main - audited_sha: 8fe913e461d45a1b2d3d0f3d5ecfe1d2b3210996 + audited_sha: 1a9d598d2ba9cfcc6704fcfd41677728bcb223c2 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 56d5fbef3..26dc11aea 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8fe913e461d45a1b2d3d0f3d5ecfe1d2b3210996`. The canonical machine baseline +1a9d598d2ba9cfcc6704fcfd41677728bcb223c2`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e131e1551..601f2e7f2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8fe913e4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1a9d598d` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3e6bb7bbe..7f85e1cf2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8fe913e4` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 1a9d598d` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6c61ea057..89007ccae 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8fe913e4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1a9d598d` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c2d87a669..4049eb9b6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8fe913e4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1a9d598d` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2569f9e43..18f7a742e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8fe913e4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1a9d598d` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a28847e94..6a606e9f1 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8fe913e4` 验证存在 +- 源码校验：以上路径均已在 `main @ 1a9d598d` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 265054ad0..0cbe913a9 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_api_server_guards.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288", "PR #6622", "PR #6499", "PR #6529"] +sources: ["Issue #5369", "PR #3576", "PR #4583", "PR #4718", "PR #4834", "PR #4905", "PR #4912", "PR #5085", "PR #5157", "PR #5374", "PR #5670", "PR #5682", "PR #5713", "PR #5732", "PR #5746", "PR #5752", "PR #5843", "PR #5957", "PR #6008", "PR #6138", "PR #6202", "Issue #5811", "PR #6150", "claude-workflow-starter-private@09dca46", "zuiho-kai/claude-workflow-starter@c217fc6", .pre-commit-config.yaml, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/diffusion_request_utils.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/entrypoints/openai/tts_adapters/, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/cfg_companion_tracker.py, vllm_omni/metrics/prometheus.py, tests/dfx/reliability/test_reliability_qwen3_omni.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_async_omni.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_api_server_guards.py, tests/entrypoints/openai_api/test_audex_serving_guards.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_tts_detection.py, tests/entrypoints/openai_api/test_video_api_utils.py, tests/entrypoints/openai_api/test_video_server.py, tests/tools/test_check_tts_adapter.py, tools/pre_commit/check_tts_adapter.py, "PR #4795", "PR #4755", "PR #3805", "PR #5878", "PR #6070", "PR #6122", "PR #4499", "PR #6050", "PR #6329", "PR #5999", "PR #5445", "PR #6288", "PR #6622", "PR #6499", "PR #6529", "PR #6776", recipes/MiniMaxAI/MiniMax-H3.md] confidence: high --- @@ -154,10 +154,10 @@ confidence: high ## SERV-1k — 非流式 MP4 编码必须按运行时能力自动选择并保留回退语义 - 触发：修改非流式 MP4 视频响应的 normalize、frame layout/dtype 判断、mux、raw/base64 输出或 fallback 行为。 -- 强制：公共 dispatcher 先完成一次帧准备和形状校验；仅当所有帧具有相同的正 HWC 形状、3/4 通道、支持的 common dtype（`uint8`、bool 或浮点）且 RGB channel view 均为 C-contiguous 时选择 `direct_planar`，逐通道量化到 PyAV 的 GBR planar 帧；其他输入必须在打开 PyAV container 前选择 `legacy_fallback`，复用已准备帧并记录稳定 fallback reason 与有效媒体参数。raw MP4 和 base64 共用该 dispatcher，不提供 CLI、模型声明或请求级编码策略；流式 fMP4 保持原路径。在线 video handler 拥有一个可复用的 8-worker `_PlanarFrameConverter`，两条非流式响应路径都传入它；utility caller 未传 converter 时以临时 serial converter 保持相同 builder 语义。 +- 强制：公共 dispatcher 先完成一次帧准备和形状校验。共同 direct 条件是所有帧具有相同的正 HWC 形状、3/4 通道和支持的 common dtype（`uint8`、bool 或浮点）。C-contiguous RGB channel view 可直接写入 PyAV GBR planar 帧；strided RGB planes 仅可由 server-owned、`max_workers > 1` 的 converter 接受，并必须复制到其 owned scratch/PyAV planes，保持相同量化语义。没有 converter 或 `max_workers == 1` 时，strided planes 必须在打开 PyAV container 前以稳定 reason `non_contiguous_rgb_planes` 选择 `legacy_fallback`。raw MP4 和 base64 共用该 dispatcher，不提供 CLI、模型声明或请求级编码策略；流式 fMP4 保持原路径。 - 强制：converter 的 scratch 是每个 worker thread 的实例本地存储；`iter_frames()` 按 FIFO 产生帧，并且一次调用最多保留 `2 * min(frame_count, max_workers)` 个 pending futures。无论 mux 成功还是抛错，调用方都必须关闭 iterator 以 cancel remaining futures；server `finally` 关闭 video handler，handler shutdown 必须幂等并以 `wait=True, cancel_futures=True` 关闭 executor。日志报告 effective frame-conversion workers（legacy 为 0）。 -- 禁止：为能力探测再次 normalize/copy 视频；在 direct mux 开始后因异常重试 legacy；把 interleaved、非法形状或不支持 dtype 静默送入 planar path；不得把 async route 仍同步执行 mux/base64 或单请求 pending 上限说成全请求 nonblocking 或所有并发请求的全局 pending 上限。共享 pool 只在一个 API process 内限制 conversion threads；多 process 仍各自拥有 pool，且性能证据仍限单请求。 -- 验收：覆盖 channel-first-backed direct path、interleaved/unsupported shape/dtype fallback、RGB contiguity、GBR plane 与 padded stride、serial/parallel exact output、FIFO 与单调用 future bound、iterator close 后 cancel、reused executor 的幂等 shutdown、direct failure 不重试；音频存在时 serving resolver 才以 24 kHz 为缺省，显式采样率和 video-only 日志保持正确。固定输入下有无音频的 direct/legacy MP4 必须 byte-identical 并通过 ffprobe/完整 FFmpeg decode，raw/base64 传同一 persistent converter，streaming 行为不变。^[PR #6288] ^[PR #6499] +- 禁止：为能力探测再次 normalize/copy 视频；在 direct mux 开始后因异常重试 legacy；把非法形状或不支持 dtype 静默送入 planar path；不得把 async route 仍同步执行 mux/base64 或单请求 pending 上限说成全请求 nonblocking、全局并发上限、普遍模型/process/performance 改进。共享 pool 只在一个 API process 内限制 conversion threads；多 process 仍各自拥有 pool。 +- 验收：覆盖无 converter、1 worker、8 workers 的 interleaved routing，及 channel-first-backed direct、非法 shape/dtype fallback、RGB contiguity、GBR plane 与 padded stride、FIFO、future bound、iterator close 后 cancel、reused executor 幂等 shutdown、direct failure 不重试；音频存在时 serving resolver 才以 24 kHz 为缺省。固定输入下有无音频的 parallel-interleaved/legacy MP4 必须 byte-identical 并通过 ffprobe/完整 FFmpeg decode，raw/base64 传同一 persistent converter，streaming 行为不变。^[PR #6288] ^[PR #6499] ^[PR #6776] ## SERV-2e — 指标发射必须保持测量边界、缺失语义与请求唯一性 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0a93bef4a..5ca52bdac 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8fe913e4`（2026-08-30）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1a9d598d`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 71748388 \ - --to 8fe913e4 \ + --from 8fe913e4 \ + --to 1a9d598d \ --repo \ --mode report-only ``` From 448dbc81e2fd229f213fac93612e68e30dcc70be Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:04:03 +0800 Subject: [PATCH 395/465] knowledge(vllm-omni): sync 759aa4ff (PR #6824) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/components/serving/rules.md | 7 ++++--- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minimax-h3/_index.md | 6 +++++- .../vllm-omni/models/minimax-h3/rules-media.md | 16 +++++++++++++++- .../repos/vllm-omni/models/minimax-h3/rules.md | 2 +- 13 files changed, 37 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a546b7b9f..706ffee0f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8fe913e461d45a1b2d3d0f3d5ecfe1d2b3210996 + previous_audited_sha: 1a9d598d2ba9cfcc6704fcfd41677728bcb223c2 audited_ref: main - audited_sha: 1a9d598d2ba9cfcc6704fcfd41677728bcb223c2 + audited_sha: 759aa4ffebefa4b293eed6068115da823fa4fb7a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 26dc11aea..872b1d941 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1a9d598d2ba9cfcc6704fcfd41677728bcb223c2`. The canonical machine baseline +759aa4ffebefa4b293eed6068115da823fa4fb7a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 601f2e7f2..d22e81a12 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1a9d598d` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 759aa4ff` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7f85e1cf2..afe20003d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1a9d598d` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 759aa4ff` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 89007ccae..cf65b1fb2 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1a9d598d` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 759aa4ff` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4049eb9b6..2dafc98b0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1a9d598d` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 759aa4ff` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 18f7a742e..f135a1ee8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1a9d598d` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 759aa4ff` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6a606e9f1..2899ab839 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1a9d598d` 验证存在 +- 源码校验：以上路径均已在 `main @ 759aa4ff` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 0cbe913a9..965686a4b 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -106,14 +106,15 @@ confidence: high uint8 buffer 及上游 device→host 分配仍存在，不能称为 O(1) memory。 - 强制：所有 frame shape 相同；只有 rank-3 HWC 且 C=4 才去 alpha，二维 width=4 灰度不能 截断。mixed float dtype 先用 `np.result_type` 得到 common dtype，再逐帧计算，以保持 legacy - stack 的 promotion/rounding/checksum；不得原地改输入。当前 uint8 fast branch 因 normalization - 先转 float32 而实际上不可达，不能拿它作为性能或语义证明。 + stack 的 promotion/rounding/checksum；不得原地改输入。仅当输入是 `np.ndarray`、`uint8`、rank-4 + FHWC、last dim=3 且 C-contiguous 时，fast branch identity-return；其他输入一律回既有 conversion。^[PR #6824] - 验收：回归测试禁止 float path 调用 `np.stack`，并覆盖 input immutability、RGBA、width-4 grayscale、mixed float16/float32 与 exact uint8 output。性能证据必须分开报告 conversion 和 未改动的 MP4 encode：PR #5732 的可复现实验仅绑定 209×1344×768 float32 RGB、24 FPS、 ultrafast、fresh process、3 次 median/RSS 10 ms；conversion 1201.15→529.47 ms、conversion+MP4 1808.96→1235.26 ms、peak RSS 8.874→4.141 GiB、above-resident 5.393→0.660 GiB，两个 hashes - 相同。PR body 的 362-frame 与 4×MI300X E2E 是另一组观察，不能混合或泛化。^[PR #5732] + 相同。PR body 的 362-frame 与 4×MI300X E2E 是另一组观察，不能混合或泛化。另须断言 exact + eligible ndarray 返回同一对象，并以 dtype/rank/channel/contiguity controls 证明其他输入仍转换。^[PR #5732] ^[PR #6824] ## SERV-1f — TTS word timestamps 必须按 transport capability 显式门禁和传输 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5ca52bdac..29690f6d8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1a9d598d`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 759aa4ff`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 8fe913e4 \ - --to 1a9d598d \ + --from 1a9d598d \ + --to 759aa4ff \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 296c71a34..4104a42b6 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742", "PR #6666"] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742", "PR #6666", "PR #6824"] confidence: high --- @@ -26,6 +26,10 @@ confidence: high output layer 才负责编码 H.264 和 mux 成带同步音频的 MP4。FL2VA 接受首帧、尾帧或有序首尾帧，Ref2VA 支持 image-only 及有 visual reference 的 image/video/audio mixed matrix。完整计数、媒体和 API 合同见 MMH3-2a/2b。^[PR #5914] +- decoded video 在两条 H3 decode 路径中于 transfer 前准备为 contiguous `uint8` BTHWC；raw + offline frame ABI 因而从 `[0,1]` float 改为 `[0,255]` uint8，audio 不变。此布局本身不保证 + frontend direct-planar route；其 converter/fallback contract 见 `SERV-1k`，完整 H3 owner rule + 完整 owner contract 见 media rules 的 `MMH3-1o`。^[PR #6824] - 生成合同固定为 24 FPS 视频与 32 kHz 音频；官方输出 duration、named ratio、768 short-edge 与 32-pixel canvas policy 在 request validation 阶段执行，而不是由 VAE 静默修正。 - `minimax_h3_disaggregated` 是显式 deploy 选取的两 stage topology：Stage 0 的 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md index 2bb054f4f..960b56c7d 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-media.md @@ -4,12 +4,26 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5752", "PR #5829", "PR #5885", "PR #5978", "PR #6555", "PR #6688", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/model_executor/models/minimax_h3/reference_video.py, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/engine/stage_runtime.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_async_omni_engine_stage_init.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6064", "PR #6813"] +sources: ["PR #5752", "PR #5829", "PR #5885", "PR #5978", "PR #6555", "PR #6688", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/model_executor/models/minimax_h3/reference_video.py, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/engine/stage_runtime.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/inputs/data.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/engine/test_async_omni_engine_stage_init.py, tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, tests/entrypoints/openai_api/test_video_api_utils.py, "PR #6064", "PR #6813", "PR #6824"] confidence: high --- # MiniMax H3 媒体输入与精度规则 +## MMH3-1o — H3 decoded video 必须在 transfer 前定型为 contiguous uint8 + +- 触发：修改 H3 VAE decode output、worker→engine transfer、`post_process` 或 raw offline frame consumer。 +- 强制：`forward` 与 step-wise `post_decode` 两条路径都在 transfer 前按 detach→float→clamp + `[0,1]`→`*255`→round→BCTHW-to-BTHWC permute 定型为 contiguous `uint8`。单个 result 直接返回， + 多个 result 才 `cat`；audio 保持原对象/语义。`post_process` 对 exact rank-5、last-channel=3 的 + uint8 video 只做 detach、D2H 与 NumPy conversion，不得再次量化；其他输入保留旧 float/BCTHW path。 +- 禁止：不得继续承诺 raw offline `OmniRequestOutput.images[0]` 是 `[0,1]` float BCTHW；新 ABI 是 + `[0,255]` uint8 BTHWC。不得把提前准备称为 direct-planar、zero-copy、byte-identical MP4 或通用性能 + 改进；#6776 下 interleaved BTHWC 的 RGB plane 仍可能选择 legacy fallback。 +- 验收：CPU contract 逐值核对 clamp/round/permute/contiguous bytes，覆盖 single/multi、两条 decode + path、postprocess exact predicate/fallback 与 audio identity。PR #6824 的单元测试不证明 E2E/MP4； + PR benchmark 使用的 candidate 也不是 exact merge target，不能据此建立性能阈值。^[PR #6824] + ## MMH3-2a — task、reference、shape 与多输出必须作为一个输入矩阵维护 - 触发：H3 `t2va`/`fl2va`/`ref2va`、reference 计数、frame index、canvas、duration 或 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index e47a6a3bd..d412e4268 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -20,7 +20,7 @@ confidence: high | grouped QKV、fused MLP、weight loader、TP | [MMH3-1a](rules-loading.md#mmh3-1a-component-namespace-与-checkpoint-transform-必须在-active-loader-前闭合) | `minimax_h3_transformer.py::MiniMaxH3DiTModel.load_weights` → active vLLM loader | | FP8 quality、audio metric、layerwise offload | `MMH3-1b` | quantization quality test → recipe/support matrix → nightly lane | | RMSNorm、RoPE、96/128 rotary dim、fused backend | `MMH3-1c` | `MiniMaxH3Attention` → shared `RMSNorm`/`RotaryEmbedding` → platform dispatch | -| RainFusion、block sparse、video layout、NPU INT8 | [MMH3-1d](rules-rainfusion.md#mmh3-1d-rainfusion-只在已验证的-h3-video-span-上稀疏) | packed sequence → attention metadata/backend plan；quant config → prefixed linear → loader/post-load | +| RainFusion/NPU INT8；decoded video BCTHW→BTHWC uint8 | `MMH3-1d`, [MMH3-1o](rules-media.md#mmh3-1o-h3-decoded-video-必须在-transfer-前定型为-contiguous-uint8) | attention plan；H3 output prepare/postprocess → video uint8 consumer | | TRTLLM、ragged packed metadata、SAGE、Skip-Softmax、Blackwell default | `MMH3-1e` | H3 metadata/roles → TRTLLM packed trim/quant gate → platform default | | text encoder、missing q/k/v 或 gate/up、eager load bookkeeping | [MMH3-1f](rules-loading.md#mmh3-1f-text-encoder-eager-load-必须证明每个-source-shard-完整) | `encoder.py::_load_weights` → `pipeline_minimax_h3.py::load_weights` strict report | | NPU packed varlen、quadratic mask、LaserAttention、prefix K/V | `MMH3-1g` | H3 packed producer → backend capability/metadata → NPU FlashAttention fallback | From 92d47974d76f3b1d484cadcc702608e615a3aa4e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:07:39 +0800 Subject: [PATCH 396/465] knowledge(vllm-omni): sync ded89346 (PR #6830) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 25 ++++++++++++++++--- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-loader-contract.md | 13 ++++++---- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 11 files changed, 41 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 706ffee0f..4503b1b6b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1a9d598d2ba9cfcc6704fcfd41677728bcb223c2 + previous_audited_sha: 759aa4ffebefa4b293eed6068115da823fa4fb7a audited_ref: main - audited_sha: 759aa4ffebefa4b293eed6068115da823fa4fb7a + audited_sha: ded8934626aaad1a3e816c3a1d9d742efc012d93 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 872b1d941..a4e217e06 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -759aa4ffebefa4b293eed6068115da823fa4fb7a`. The canonical machine baseline +ded8934626aaad1a3e816c3a1d9d742efc012d93`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 2afe78695..973c03f95 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-05 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818"] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818", "PR #6830", tests/diffusion/conftest.py, tests/diffusion/attention/test_flash_attn.py] confidence: high --- @@ -119,7 +119,8 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload ## OMNI-CI-2b — 并行测试基础设施隔离任务失败和共享状态 -- 触发：xdist、共享下载 fixture、长寿命 judge/transcriber worker、进程池 retry，或 spawned `torch.distributed` process group。 +- 触发：xdist、共享下载 fixture、长寿命 judge/transcriber worker、进程池 retry、spawned + `torch.distributed` process group，或测试修改 PyTorch process-global default state。 - 强制：任务级异常后丢弃污染 worker；submit/result 串行，只有进程崩溃可 retry-once；传给 xdist 的参数可序列化，依赖固定兼容 major；共享文件在读锁前显式检查存在，写入保持锁语义。spawned `torch.distributed` 必须为每个 group 传入新鲜 `file://` rendezvous URL，不能先释放 TCP port 再把其数字交给子进程；`get_open_port` 只用于真实 TCP service。 - 共享 helper 的边界必须清晰：请求 client 从 `tests/helpers/client.py`、环境/设备清理由 @@ -128,10 +129,16 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload 不会调用 `__exit__`，所以必须立即回滚已启动的 stage 子进程、关闭已打开的日志 FD 并清除临时 日志；只有显式 `VLLM_OMNI_KEEP_LOG` 调试选择可保留日志文件。测试 deploy helper 只接受新 schema 的顶层 `stages`；不得静默回退 `stage_args`，缺失时 consumer error 必须指出配置文件。 + `tests/diffusion/` 的 autouse fixture 还必须在每个 test 前 snapshot `torch.get_default_dtype()`， + 并在 `finally` 恢复，使失败、skip 和正常返回采用同一 cleanup。 - 禁止：普通 task error 复用同一进程；为 pytest 生命周期不可达的竞态堆测试；把并行参数或 - 活对象隐式跨进程传递；把 file rendezvous 套用于 Mooncake/RDMA 等真实监听服务。 + 活对象隐式跨进程传递；把 file rendezvous 套用于 Mooncake/RDMA 等真实监听服务；依赖 collection + order 掩盖 `torch.set_default_dtype` 泄漏，或把 diffusion-tree fixture 称为树外测试的保护。 - 验收：OOM、普通异常和 BrokenProcessPool 分别验证隔离/重试；online/offline xdist 均通过， - 并发 cache miss 只产生一个完整文件；spawned Gloo/NCCL group 使用独立 file rendezvous，真实 TCP service 仍使用 port helper。 ^[PR #6208] ^[PR #6339] ^[PR #6468] ^[PR #6523] + 并发 cache miss 只产生一个完整文件；spawned Gloo/NCCL group 使用独立 file rendezvous，真实 TCP + service 仍使用 port helper。另以 dtype-mutating predecessor + default-sensitive F32 test 覆盖正常与 + exception cleanup。该 fixture 只覆盖 `tests/diffusion/`；Ovis 的原始 leak 若在树外运行，仍须在 + source 修复。^[PR #6208] ^[PR #6339] ^[PR #6468] ^[PR #6523] ^[PR #6830] ## OMNI-CI-2c — 昂贵 engine fixture 复用必须恢复状态并隔离拓扑 @@ -212,6 +219,16 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload have explicit timeouts and the unresolved TTS case is non-blocking. On ROCm, check the selected test's per-run baseline/steady-state sampling and cleanup command separately from the model assertion; a local or different machine threshold result is not a CI-wide performance verdict. ^[PR #6704] +- 强制：MI300 blocking layerwise-offload step 只 deselect exact + `stabilityai/stable-audio-open-1.0` parameter；同一 case 在独立 `mi300_1`、依赖 `amd-build` 的 + `NonBlocking` step 运行，其余 test parameters、spawn/log env 保持。ROCm BF16 AITER + FlashAttention-vs-SDPA cross-attention 可用 elementwise `assert_close(rtol=1e-2, atol=1.5e-2)`； + CUDA/XPU 仍保留各自 strict checks。 +- 禁止：用邻近 shard 或绿色 NonBlocking case 宣称 Stable Audio memory-savings issue 已解决；不得把 + ROCm tolerance 推广到其他平台、attention path、输出质量或 runtime parity。 +- 验收：render exact deselection node ID、quarantine command、dependency 与 `NonBlocking` wiring，且 + blocking test file 保留其他 parameters。ROCm test 保留 masked/unpadded cross-attention coverage，并 + 以超出 rtol/atol 的 mutation 证明仍会失败。^[PR #6830] ## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d22e81a12..7381cce1d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 759aa4ff` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ded89346` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index afe20003d..93dbd4f30 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 759aa4ff` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ ded89346` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index cf65b1fb2..132656442 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 759aa4ff` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ded89346` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2dafc98b0..1555411cd 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 759aa4ff` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ded89346` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md index d05f1955f..8564507c6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py", "PR #6138", "PR #6640", vllm_omni/engine/stage_init_utils.py, tests/engine/test_async_omni_engine_stage_init.py, tests/engine/test_stage_engine_args.py] +sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py", "PR #6138", "PR #6640", vllm_omni/engine/stage_init_utils.py, tests/engine/test_async_omni_engine_stage_init.py, tests/engine/test_stage_engine_args.py, "PR #6830", vllm_omni/engine/arg_utils.py, tests/engine/test_arg_utils.py] confidence: high --- @@ -86,12 +86,15 @@ confidence: high weight artifact 与 index 引用的全部 shards 完整时，才可改写为本地 subdir；partial cache 必须按同一 repo/revision/download_dir 请求精确 subset 并复验。仅当 `tokenizer_subdir is not None` 且 `tokenizer_revision != revision` 时单独解析 tokenizer snapshot；空 tokenizer subdir 表示 snapshot - root，仍须验证 vocabulary artifact。`is_runai_obj_uri` 命中时保留 URI 与 joined subpath，跳过 + root，仍须验证 vocabulary artifact。remote tokenizer subfolder 成功解析为本地目录后，success + logging 与 `self.tokenizer` assignment 都必须使用同一个 resolved `tokenizer_subfolder`。`is_runai_obj_uri` 命中时保留 URI 与 joined subpath，跳过 local/HF snapshot 推断，交给 upstream streamer。 - 禁止：以 `isdir`、config/index 文件存在或单 shard 证明 stage 完整；partial snapshot 静默变成本地 model path；把显式 tokenizer revision 改回 model revision；把 Run:AI URI 当作本地/HF cache，或把 - generic stage resolver 描述成会补齐模型专有的全部 composite root components。 + generic stage resolver 描述成会补齐模型专有的全部 composite root components；不得因 success-path + log 引用未定义变量，把已下载 subfolder 捕获为 warning/failure。 - 验收：覆盖 cold/partial/warm offline stage snapshot、index 缺 referenced shard、root tokenizer、 - split tokenizer revision、download failure fail-closed 和 Run:AI URI passthrough；模型专有 root + split tokenizer revision、download failure fail-closed 和 Run:AI URI passthrough；mocked CosyVoice3 + success 还须断言 resolved tokenizer path 被赋值且 warning logger 未调用。模型专有 root completeness 另由 owner 验证。unit fixture 不证明真实 checkpoint load、音质、GPU E2E 或跨平台支持。 - ^[PR #6640] + ^[PR #6640] ^[PR #6830] diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f135a1ee8..b23cdd413 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 759aa4ff` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ded89346` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2899ab839..40879043f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 759aa4ff` 验证存在 +- 源码校验：以上路径均已在 `main @ ded89346` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 29690f6d8..3c662ee0a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 759aa4ff`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ded89346`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 1a9d598d \ - --to 759aa4ff \ + --from 759aa4ff \ + --to ded89346 \ --repo \ --mode report-only ``` From 57c3a7e0d49774bfd356b9fe1757fea31c7db948 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:09:02 +0800 Subject: [PATCH 397/465] knowledge(vllm-omni): sync 4cca28e9 (PR #6832) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4503b1b6b..a948ee1fc 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 759aa4ffebefa4b293eed6068115da823fa4fb7a + previous_audited_sha: ded8934626aaad1a3e816c3a1d9d742efc012d93 audited_ref: main - audited_sha: ded8934626aaad1a3e816c3a1d9d742efc012d93 + audited_sha: 4cca28e9bd96c2ae4758c90c61eaf86dff82c8fd # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a4e217e06..215904e99 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ded8934626aaad1a3e816c3a1d9d742efc012d93`. The canonical machine baseline +4cca28e9bd96c2ae4758c90c61eaf86dff82c8fd`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 7381cce1d..d9a680ea8 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ded89346` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4cca28e9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 93dbd4f30..5d6ee8816 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ded89346` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 4cca28e9` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 132656442..aed02259f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ded89346` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4cca28e9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1555411cd..c951beca7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ded89346` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4cca28e9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b23cdd413..eee3efd39 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ded89346` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4cca28e9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 40879043f..7bea24892 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ded89346` 验证存在 +- 源码校验：以上路径均已在 `main @ 4cca28e9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3c662ee0a..37f6f76c7 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ded89346`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4cca28e9`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 759aa4ff \ - --to ded89346 \ + --from ded89346 \ + --to 4cca28e9 \ --repo \ --mode report-only ``` From 31e4fb2a174021859a1a985ec1a56e39502e6b70 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:11:53 +0800 Subject: [PATCH 398/465] knowledge(vllm-omni): sync f321cdbe (PR #6628) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minicpm-o-4-5/_index.md | 5 ++++- .../models/minicpm-o-4-5/rules-code2wav-batching.md | 12 ++++++++++-- .../repos/vllm-omni/models/minicpm-o-4-5/rules.md | 1 + 12 files changed, 27 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a948ee1fc..e3b62660c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ded8934626aaad1a3e816c3a1d9d742efc012d93 + previous_audited_sha: 4cca28e9bd96c2ae4758c90c61eaf86dff82c8fd audited_ref: main - audited_sha: 4cca28e9bd96c2ae4758c90c61eaf86dff82c8fd + audited_sha: f321cdbe16df69da4f1770decc500ad55ed764ac # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 215904e99..f89424609 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4cca28e9bd96c2ae4758c90c61eaf86dff82c8fd`. The canonical machine baseline +f321cdbe16df69da4f1770decc500ad55ed764ac`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d9a680ea8..8bc8f99e2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4cca28e9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ f321cdbe` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5d6ee8816..e62ba8601 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4cca28e9` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ f321cdbe` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index aed02259f..cac7a0afe 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4cca28e9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ f321cdbe` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c951beca7..fb0ed2668 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4cca28e9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ f321cdbe` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index eee3efd39..2caf2f626 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4cca28e9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ f321cdbe` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7bea24892..0961079ad 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4cca28e9` 验证存在 +- 源码校验：以上路径均已在 `main @ f321cdbe` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 37f6f76c7..5b71dcbc3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4cca28e9`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ f321cdbe`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from ded89346 \ - --to 4cca28e9 \ + --from 4cca28e9 \ + --to f321cdbe \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md index 090b3b384..7541f8136 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: index tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6404", "PR #6678", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082", "PR #6587"] +sources: ["PR #3642", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6404", "PR #6678", "PR #6628", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, "PR #6082", "PR #6587"] confidence: high --- @@ -55,6 +55,9 @@ Talker chunk、shipping profile 与 chat 隔离见 [native duplex 规则](rules- state、batch cap 与 NPU residual limits 见 [Code2Wav 并发批处理规则](rules-code2wav-batching.md)。新模型语义验证见 [model validation](../../review/guides/model-validation.md)。 +Seed-TTS chat reference audio 的 request lookup、private prompt preservation、Talker buffer 与 +Code2Wav per-field fallback 见 MCPMO-3e；它不覆盖 realtime。^[PR #6628] + Realtime video 不以「每 200 ms append」定义 frame cadence：首个约 1030 ms、以后每秒关闭的 Stage 0 unit 才接收对应 camera frame；双图 stacking 与外部 WAV 覆盖 soundtrack 的边界见本页已链接的 native duplex 规则 MCPMO-4f。^[PR #6404] ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-code2wav-batching.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-code2wav-batching.md index 349af310d..b1dfb1ed1 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-code2wav-batching.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-code2wav-batching.md @@ -1,15 +1,23 @@ --- title: "MiniCPM-o 4.5 Code2Wav 并发批处理规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6021", vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/platforms/npu/test_graph_tools.py] +sources: ["PR #6021", "PR #6628", vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py, tests/entrypoints/openai_api/test_serving_chat_minicpmo45_reference_audio.py, tests/model_executor/models/minicpmo_4_5/test_llm2tts.py, tests/model_executor/models/minicpmo_4_5/test_minicpmo_4_5_async_chunk.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/platforms/npu/test_graph_tools.py] confidence: high --- # MiniCPM-o 4.5 Code2Wav 并发批处理规则 +## MCPMO-3e — MiniCPM-o 4.5 reference audio 必须经过私有 prompt 到 Code2Wav handoff + +- 触发：MiniCPM-o 4.5 chat `ref_audio`、Seed-TTS、`llm2tts`、Code2Wav prompt selection 或 async chunk。 +- 强制：仅 `_has_minicpmo45_stage` 时读取 reference，优先级为 root attr → `extra_body` → `model_extra.extra_body` → `model_extra`；absent no-op，non-string `ValueError`。chat rendering/policy 后用 `MediaConnector` decode 一次，写回 original prompt private key，不复制 message。`llm2tts` 优先普通 multimodal audio，否则写 private key 至 runner-owned `model_intermediate_buffer.codes.ref` / `meta.ref_audio_sr`。 +- 强制：Code2Wav 对每个 named section 先取 legacy `additional_information`，再以 buffer 中实际提供的 field 覆盖，缺 field 仍 fallback；无 ref 保持 HT default。async 只在 first chunk/cache epoch handoff。 +- 禁止：不得为非 MiniCPM 解码、重新序列化 Seed-TTS ref、或将该 chat path 外推至 realtime、通用模型或质量结论。 +- 验收：覆盖 root/nested precedence、missing/non-string、一次 decode/serialization、ordinary audio precedence、private buffer/per-field legacy fallback、default HT、async first-chunk/cache epoch；CPU chain mock 不替代 engine E2E。单个 MI300X Seed-TTS row 只证明该 exact request 的 `runtime-ref-*` selection；AMD CI #11229 失败，且 realtime 仍 deferred。^[PR #6628] + ## MCPMO-1j — Code2Wav 并发 duplex batch 必须按 request 保存 ragged state - 触发：修改 `MiniCPMO45Code2Wav` bucket/initial-batch partition、`BatchedToken2Wav.decode_ragged_batch`、attention cache dtype 或 NPU Code2Wav patch。 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 3ab516fce..9753dff91 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -25,6 +25,7 @@ confidence: high | HiFT CUDA Graph、chunk bucket、lazy capture、iSTFT | [`MCPMO-1d`](rules-cuda-graphs.md#mcpmo-1d-hift-graph-只捕获稳定-pre-istft-子图并限界-shape-cache) | `cuda_graph_wrapper.py::HiFTGraphWrapper` → `BatchedToken2Wav._hift_inference` → shared `HiFTGenerator` | | CFM DiT CUDA Graph、shape cache、generation retirement、eager fallback | [`MCPMO-1e`](rules-cuda-graphs.md#mcpmo-1e-cfm-dit-cuda-graph-必须按-shape-整代退休并保持-eager-parity) | `cuda_graph_wrapper.py::CFMGraphWrapper` → `BatchedToken2Wav._estimator_step` | | batch、`runtime_info`、stage handoff | MCPMO-3a/3b | `stage_input_processors/minicpmo_4_5_omni.py` → `minicpmo_4_5_omni.py::MiniCPMO45OmniForConditionalGeneration` → TTS/code2wav | +| Seed-TTS、chat `ref_audio`、runtime-ref、Code2Wav handoff | [MCPMO-3e](rules-code2wav-batching.md#mcpmo-3e-minicpm-o-45-reference-audio-必须经过私有-prompt-到-code2wav-handoff) | serving chat MediaConnector → original prompt private key → `llm2tts` buffer → Code2Wav | | Talker codec sampling、repetition penalty、request RNG/compaction | MCPMO-3c | `minicpmo_4_5_omni_tts.py::{make_omni_output,_sample_audio_codes,_apply_batched_repetition_penalty}` → `test_talker_batching.py` | | native duplex、Stage0 resume、LISTEN/SPEAK、server VAD | MCPMO-4a | `experimental/fullduplex/{minicpmo45,openai}/` → stage input processor | | instructions/native-mode update、first-append reservation、context lock | [`MCPMO-4b`](rules-duplex.md#mcpmo-4b-native-duplex-session-update-必须如实拒绝不可重建的上下文) | `experimental/fullduplex/minicpmo45/{adapter,session}.py` → runtime adapter/session runner | From 9dd9a54e412abb49c99991365eadf1941dff283e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:16:17 +0800 Subject: [PATCH 399/465] knowledge(vllm-omni): sync 5e06d2e7 (PR #6831) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/rules.md | 10 ++++++---- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/nemotron-voicechat/rules.md | 11 ++++++----- 11 files changed, 24 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e3b62660c..c403e3e58 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4cca28e9bd96c2ae4758c90c61eaf86dff82c8fd + previous_audited_sha: f321cdbe16df69da4f1770decc500ad55ed764ac audited_ref: main - audited_sha: f321cdbe16df69da4f1770decc500ad55ed764ac + audited_sha: 5e06d2e73c944fdfae7767b36557f4966ea2069b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f89424609..f0e7373b1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -f321cdbe16df69da4f1770decc500ad55ed764ac`. The canonical machine baseline +5e06d2e73c944fdfae7767b36557f4966ea2069b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8bc8f99e2..49493912c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ f321cdbe` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5e06d2e7` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e62ba8601..701a28e8a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ f321cdbe` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 5e06d2e7` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index cac7a0afe..db06bd1fd 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ f321cdbe` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5e06d2e7` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index fb0ed2668..ca39f3d03 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ f321cdbe` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5e06d2e7` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2caf2f626..e658521b2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ f321cdbe` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5e06d2e7` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/scheduler/rules.md b/knowledge/repos/vllm-omni/components/scheduler/rules.md index 309827e6e..dfe230cfb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/rules.md +++ b/knowledge/repos/vllm-omni/components/scheduler/rules.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-05 type: rule tags: [vllm-omni, components, scheduler] -sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/core/sched/test_omni_scheduler_mixin_timeouts.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619", "PR #6680", "PR #6626", "PR #6529", tests/core/sched/test_omni_ar_scheduler_aborted_queue_sweep.py, tests/core/sched/test_omni_scheduler_streaming_input_counter.py, tests/core/sched/test_omni_sched_deferred_free_fence.py] +sources: ["PR #5957", "PR #5976", tests/core/sched/test_omni_ar_scheduler_stale_drain.py, tests/core/sched/test_omni_ar_scheduler_streaming.py, "vllm-omni-rebase-agent@122a9468:agent/skills/fix-talker-truncated-prefill-prefix-cache-key-cap/SKILL.md", "vllm-omni-rebase-agent@122a9468:agent/skills/gpu-hang-low-max-num-batched-tokens/SKILL.md", vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/core/prefix_cache.py, vllm_omni/utils/mm_outputs.py, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_generation_scheduler.py, vllm_omni/core/sched/omni_scheduler_mixin.py, vllm_omni/core/sched/omni_scheduling_coordinator.py, vllm_omni/core/sched/output.py, tests/core/test_prefix_cache.py, tests/core/test_prefix_cache_async_write.py, tests/core/sched/test_omni_scheduler_mixin_shared.py, tests/core/sched/test_omni_scheduler_mixin_timeouts.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/utils/test_mm_outputs.py, tests/entrypoints/test_omni_new_request_data.py, "PR #4106", "PR #5310", "PR #5461", "PR #4795", "PR #5842", "PR #6021", "PR #6033", "PR #6089", "PR #6149", "PR #6360", "PR #6406", "PR #6150", "PR #6619", "PR #6680", "PR #6626", "PR #6529", tests/core/sched/test_omni_ar_scheduler_aborted_queue_sweep.py, tests/core/sched/test_omni_scheduler_streaming_input_counter.py, tests/core/sched/test_omni_sched_deferred_free_fence.py, "PR #6831", tests/e2e/online_serving/test_nemotron_voicechat_duplex.py] --- # Scheduler 规则 @@ -260,10 +260,12 @@ modules=[online_serving, worker_runner]，status=active，run_count=38，2026-06 ## SCHED-5f — async-chunk 等待必须有可配置的截止时间 -- 触发：async-chunk 请求进入 `WAITING_FOR_CHUNK`，或部署配置读取 `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S`。 +- 触发：async-chunk 请求进入 `WAITING_FOR_CHUNK`，或配置/测试设置 `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S`。 - 强制：统一校验超时值，拒绝负数和非有限值，`0` 仅作为显式禁用并记录 warning；chunk adapter 在请求停等时开始计时、每次收到 chunk 后重置，并由 scheduler 及时结束超时且仍在 `self.requests` 中的请求。 -- 禁止：只保护 full-payload 的 `WAITING_FOR_INPUT`；让负数、`nan` 或 `inf` 静默关闭 deadline；按 stream 总生命周期而非 chunk 间 stall 时间计时；对已离开 scheduler 的请求调用 `finish_requests`。 -- 验收：覆盖有效值、显式零值、负数和非有限值解析，覆盖 dropped terminal chunk、producer 无响应、chunk 到达后重置、请求完成/abort 清理及同批健康请求继续运行；断言超时请求为 `FINISHED_ERROR`，并确认模块 reload 不污染后续测试。^[PR #6033] +- 强制：stall diagnostic 在 scheduler import 前设置 `0 < server wait < client wait`。 +- 禁止：只保护 full-payload 的 `WAITING_FOR_INPUT`；让负数、`nan` 或 `inf` 静默关闭 deadline；按 stream 总生命周期计时；对已离开 scheduler 的请求调用 `finish_requests`；把 server error 当 client delivery/生产预算。 +- 验收：覆盖解析、stall/reset、完成/abort 与健康同批请求；超时为 `FINISHED_ERROR`。Nemotron 固定 + `240 < 300`，server request-ID/error 先于 client timeout，健康 control 两者皆不触达。^[PR #6033] ^[PR #6831] ## SCHED-5g — resumable async-chunk 终态清理必须以 live queue 所有权为准 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 0961079ad..8b911977d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ f321cdbe` 验证存在 +- 源码校验：以上路径均已在 `main @ 5e06d2e7` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5b71dcbc3..376154245 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ f321cdbe`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5e06d2e7`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 4cca28e9 \ - --to f321cdbe \ + --from f321cdbe \ + --to 5e06d2e7 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md b/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md index b2ef220c2..28a04e72e 100644 --- a/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md +++ b/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md @@ -1,10 +1,10 @@ --- title: "Nemotron VoiceChat 规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, serving] -sources: ["PR #6089", vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/experimental/fullduplex/openai/, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py, tests/e2e/features/fullduplex/nemotron_voicechat/, tests/e2e/online_serving/test_nemotron_voicechat_duplex.py] +sources: ["PR #6089", vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/experimental/fullduplex/openai/, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py, tests/e2e/features/fullduplex/nemotron_voicechat/, tests/e2e/online_serving/test_nemotron_voicechat_duplex.py, "PR #6831"] confidence: high --- @@ -35,7 +35,8 @@ confidence: high ## NVC-1d — full-model 音频证据和部署声明必须限界 -- 触发：修改 full-model E2E gate、Buildkite selection 或 duplex profile。 -- 强制：full-model probe 除 response/event 完成外，至少断言 PCM RMS floor、最小 audio packet 数和与 input frame 数相关的 duration/bytes；offline streaming 要比较其原有完整 audio contract。profile 的 `max_num_seqs: 1` 只证明默认 admission 配置。 -- 禁止：用非空 bytes、约 1 LSB 的 RMS 或文本成功证明音频可用；把作者在 H200 实测、generic nightly selector 或 YAML 存在写成 Buildkite pass、wall-clock realtime、确定性 barge-in 或跨硬件保证。 +- 触发：修改 full-model E2E gate、Buildkite selection、duplex profile 或 stall-diagnostic timeout ordering。 +- 强制：full-model probe 除 response/event 完成外，至少断言 PCM RMS floor、最小 audio packet 数和与 input frame 数相关的 duration/bytes；offline streaming 要比较其原有完整 audio contract。profile 的 `max_num_seqs: 1` 只证明默认 admission 配置。stall diagnostic 用 test-local server 240 秒 deadline 与 client 300 秒 wait，使 stage diagnostics 先于 client expiry；healthy run 不应触达任一 deadline。 +- 禁止：用非空 bytes、约 1 LSB 的 RMS 或文本成功证明音频可用；把作者在 H200 实测、generic nightly selector 或 YAML 存在写成 Buildkite pass、wall-clock realtime、确定性 barge-in 或跨硬件保证。不得把 240 秒写成生产模型设置、把 `FINISHED_ERROR` 称作 client error delivery，或声称 #6816 已修复。 - 验收：PR 作者报告 targeted H200 suite 306 passed，reviewer 后续报告 399 CPU tests、offline WAV byte-identical 和 native duplex non-silent audio；这些是提交时证据，当前 KB validation 不能复现真实 checkpoint/H200 E2E。Buildkite 改动只把既有 AMD-ready 步骤的超时从 20 分钟增至 45 分钟，不新增该模型专属 pipeline。^[PR #6089] +- stall diagnostic 的验收固定 server 240/client 300，复现 stall 时 server 先记录 full request ID 并标记 `FINISHED_ERROR`，之后 bare client `TimeoutError` 才到期；同时正常 run 仍有 audio。reviewer 已复现该 ordering，仍未证明 client error event。^[PR #6831] From 76c6b5da4e62d2cdaaf255743d4a47193d8a9c61 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:23:49 +0800 Subject: [PATCH 400/465] knowledge(vllm-omni): sync 86b85c07 (PR #6714) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-system-runtime.md | 4 +-- .../components/diffusion/sigma-schedules.md | 6 ++--- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/minimax-h3/_index.md | 4 ++- .../models/minimax-h3/rules-cache-task.md | 25 +++++++++++++++++-- .../vllm-omni/models/minimax-h3/rules.md | 1 + 14 files changed, 44 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c403e3e58..e4d031026 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: f321cdbe16df69da4f1770decc500ad55ed764ac + previous_audited_sha: 5e06d2e73c944fdfae7767b36557f4966ea2069b audited_ref: main - audited_sha: 5e06d2e73c944fdfae7767b36557f4966ea2069b + audited_sha: 86b85c078bc041e04aee4c4d9167fb10fb1994c7 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f0e7373b1..31530acfb 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5e06d2e73c944fdfae7767b36557f4966ea2069b`. The canonical machine baseline +86b85c078bc041e04aee4c4d9167fb10fb1994c7`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 49493912c..2fbb7058c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5e06d2e7` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 86b85c07` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 701a28e8a..26e471baa 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5e06d2e7` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 86b85c07` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index 391a092e4..db085219e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/distributed/test_a2a_permute.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/distributed/test_a2a_permute.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] confidence: high --- @@ -83,7 +83,7 @@ confidence: high ## DIFF-4x — SymmMem Ulysses A2A 只能显式启用并保持 workspace 生命周期闭合 - 触发：修改 `ulysses_a2a_permute`、strict Ulysses all-to-all、NCCL SymmetricMemory JIT extension、CUDA graph capture 或 diffusion worker shutdown。 -- 强制：默认关闭；只在 `ulysses_degree > 1`、strict Ulysses 的正向 `(scatter,gather)=(2,1)` 与反向 `(1,2)` layout 启用，并在 worker/model 初始化时 JIT build/capability failure 直接 fail-at-init。advanced-UAA 保持其 `_ulysses_all_to_all_any_*` path；AllGather 不选择 Ulysses；strict Ulysses+Ring 的 Ulysses leg 仍可使用该 path。每个 `(device, process-group)` 只有一个单 CUDA stream 的 grow-only byte workspace；首次分配及增长必须是 collective、在增长前同步，容量内按 typed view 复用；graph capture 中禁止增长，须先 warm up 最大 shape；worker shutdown 同步并释放 workspace。 +- 强制：默认关闭；只在 `ulysses_degree > 1`、strict Ulysses 的正向 `(scatter,gather)=(2,1)` 与反向 `(1,2)` layout 启用，并在 worker/model 初始化时 JIT build/capability failure 直接 fail-at-init。advanced-UAA 保持其 `_ulysses_all_to_all_any_*` path；AllGather 不选择 Ulysses；strict Ulysses+Ring 的 Ulysses leg 仍可使用该 path。每个 `(device, process-group)` 只有一个单 CUDA stream 的 grow-only byte workspace；首次分配及增长必须是 collective、在增长前同步，容量内以 typed byte-slice view 复用；graph capture 中禁止增长，须先 warm up 最大 shape；worker shutdown 同步并释放 workspace。^[PR #6714] - 禁止：把 flag 当作默认 transport、静默 fallback；按历史 shape/dtype 无界缓存 allocation；跨 stream 重用 staging buffer；capture 中隐式分配/扩容；把 CPU/mock workspace 测试或 PR benchmark 写成数学/输出 parity、普遍速度或跨硬件收益。 - 验收：覆盖 CLI/deploy/default-stage 的显式 flag 透传与默认 false、eligible strict path 的 init build、UAA/AllGather/noneligible layout 不调用该 path、strict Ulysses+Ring 的 eligible Ulysses leg、peak workspace 复用/collective growth、capture growth fail、single-stream reject 与 shutdown release；真实多 rank hardware 仍须对 regular Ulysses 做 Q/K/V 与 reverse-output parity，再单独记录精确 shape/stack/预热后的性能。^[PR #6340] diff --git a/knowledge/repos/vllm-omni/components/diffusion/sigma-schedules.md b/knowledge/repos/vllm-omni/components/diffusion/sigma-schedules.md index 6925fce01..bb65cd4fa 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/sigma-schedules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/sigma-schedules.md @@ -1,10 +1,10 @@ --- title: "Diffusion distilled sigma schedule 规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion, scheduler] -sources: ["PR #5991", vllm_omni/diffusion/models/dmd2/config.py, vllm_omni/diffusion/models/dmd2/mixin.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py] +sources: ["PR #5991", "PR #6714", vllm_omni/diffusion/models/dmd2/config.py, vllm_omni/diffusion/models/dmd2/mixin.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/sched/test_dmd2_sigma_schedule.py] confidence: high --- @@ -14,7 +14,7 @@ confidence: high - 触发：新增 distilled/DMD2 checkpoint schedule metadata、变更 sigma shift 或把 `DMD2SigmaSchedule` 接入新 pipeline。 -- 强制：continuous `base_schedule` 至少两个有限数，严格从 1.0 递减到 +- 强制：continuous `base_schedule` 至少两个有限数，严格从有限 `(0,1]` 首点递减到 0.0；先拒绝非有限值，再逐对验证相邻位置严格单调。区间数 `len(boundaries)-1` 才是 request/cache/denoise step count，solver 保留全 boundary list。 - 强制：`from_metadata()` 的 absent key 与 explicit null 都因 `metadata.get()` 返回 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index db06bd1fd..eda2fe283 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5e06d2e7` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 86b85c07` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ca39f3d03..4fefb8a07 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5e06d2e7` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 86b85c07` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e658521b2..e3c2fa53e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5e06d2e7` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 86b85c07` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8b911977d..26863f7af 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5e06d2e7` 验证存在 +- 源码校验：以上路径均已在 `main @ 86b85c07` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 376154245..f50d63c7f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5e06d2e7`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 86b85c07`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from f321cdbe \ - --to 5e06d2e7 \ + --from 5e06d2e7 \ + --to 86b85c07 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 4104a42b6..528555ccd 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742", "PR #6666", "PR #6824"] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742", "PR #6666", "PR #6824", "PR #6714"] confidence: high --- @@ -96,6 +96,8 @@ confidence: high 交互与单 H100 校准边界见 MMH3-2g。 - distilled checkpoint 的 continuous RF `base_schedule`、四步语义与 partition 隔离见 MMH3-2h；它不替换 H3 的自定义 solver。 +- FastH3 是 exact FastVideo artifact 的 startup-only t2va fusion；它不使用 Turbo/native dynamic + manager，strict artifact/request/offload gates 见 MMH3-2n。^[PR #6714] - step execution 是 H3 的模型专有 stateful path：只在 resolved attention backend 能隔离多个 packed documents 时合并 request；否则仍逐 request 前向。request state、rank-0 preparation、Cache-DiT/DLO/multi-output 排除与有界验证证据见 MMH3-2k，不能由 generic runner guard 推断 multi-rank failure synchronization。^[PR #5810] - LightX2V Turbo v1.0 仅支持 legacy dynamic LoRA；精确 artifact mapping、packed QKV/FC1 binding、active-request sampling 合同与 DLO-resident A/B sidecar 边界见 MMH3-2j；转换后的 mixed-rank diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md index 94fe92665..dd8941af5 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", "PR #6550", "PR #6666", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/npu/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_native_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py] +sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", "PR #6550", "PR #6666", "PR #6714", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/fasth3.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/npu/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/entrypoints/openai/video_api_utils.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_fasth3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_native_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py, tests/entrypoints/openai_api/test_video_api_utils.py] confidence: high --- @@ -132,7 +132,8 @@ Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期 - 强制：缺少 metadata key 或显式 null 都表示未蒸馏，保持旧 `num_inference_steps or 50` 的 uniform-point 构造（默认 50 个点实际是 49 个 solver intervals）；explicit empty list 必须拒绝。调度至少两个有限点，严格 - 从 1.0 递减到 0.0，并逐对验证相邻位置。 + 从有限 `(0,1]` 首点递减到 0.0，并逐对验证相邻位置；FastH3 的首点是 `0.999`，不能被 + 旧的 exact-1.0 假设拒绝。 - 强制：5 个 sigma boundaries 只有 4 个 denoise intervals；`num_inference_steps`、 Cache-DiT quality policy 和请求验证均使用 `len(base_schedule)-1`，solver 仍接收 完整 boundary list。用户省略 step 时自动采用 checkpoint 步数；显式值只有 @@ -154,6 +155,26 @@ Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期 matching/mismatched explicit step 和 legacy/partially-constructed fallback。仍缺真实 distilled checkpoint 的 CUDA/NPU E2E 与 teacher/student 质量阈值。^[PR #5991] +## MMH3-2n — FastH3 只接受精确 artifact 并在启动时一次性融合 + +- 触发：FastH3 `lora_path`、load-time fusion、four-step request 或 serving gate。 +- 强制：只 claim `fastvideo-lora-v2`、`finetuned_model` 位于 `fastvideo/` 且 identity 含 `fasth3` + 的 exact FastH3 artifact；`base_model` 可缺省，声明时必须精确为 `MiniMaxAI/MiniMax-H3`。bundle + root 与 claimed variant 都必须唯一，否则 fail closed；普通 FastVideo/PEFT/多 shard 交还 legacy + dynamic LoRA，不可误认。 +- 强制：声明的 low-rank/diff/`set_weight` tensor count 必须 numeric 且与实物一致；任何 unmapped、 + duplicate/unpaired factor、shape mismatch、extra/incomplete block、VSA `set_weight` 或 fused QKV/FC1 + 不支持的 full-rank delta 都失败。fusion 在 sharding 前完成：Diffusers Q/K/V 重排为 grouped native + QKV，FC1 value-first 重排为 gate-first，再加入 low-rank `B@A` 与 full-rank `diff`/`diff_b`。成功后 + 只保留永久 fused sentinel；不能 request-switch，dynamic manager 的 list/add/remove/pin 安全返回空/false。 +- 强制：serving 只允许 `t2va`、无 per-request `lora`、exact `num_inference_steps=4`（五 boundaries/ + 四 forwards）以及默认 video/audio shift `12/3`；拒绝 CPU/layerwise/DLO offload 与 Ref2VA。 +- 禁止：不得混同 Turbo/native dynamic adapters，支持 VSA/multi-adapter/Ref2VA/offload，或声称 + quality parity、平台 portability 与通用性能收益；recipe/body 的旧 steps=5 不能覆盖 source 的 4。 +- 验收：synthetic artifact 覆盖 identity/base/root/variant、metadata counts、mapping/pairs、QKV/FC1 + reorder、full/low-rank delta、block/shape 与 unapplied patch failures；request matrix 覆盖 task、steps、 + shifts、request LoRA 和各 offload gate。CPU tests 不替代真实 artifact/hardware 质量或性能验证。^[PR #6714] + ## MMH3-2j — Turbo LoRA 只接受精确发布 artifact，并与 H3 task、sampling 和 offload lifecycle 共同门禁 - 触发：MiniMax-H3 LightX2V Turbo 的 dynamic LoRA loader、artifact conversion、request task/steps/flow shift，或与 CPU、standard layerwise、distributed layerwise offload (DLO) 的组合。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index d412e4268..b74a793b8 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -38,6 +38,7 @@ confidence: high | distilled、DMD2、`base_schedule`、4-step、sigma boundary | [MMH3-2h](rules-cache-task.md#mmh3-2h-distilled-sigma-schedule-按-partition-所有且以区间计步) | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | | LightX2V Turbo、legacy dynamic LoRA、packed QKV/FC1、five sigma points、DLO resident A/B sidecars | [MMH3-2j](rules-cache-task.md#mmh3-2j-turbo-lora-只接受精确发布-artifact并与-h3-tasksampling-和-offload-lifecycle-共同门禁) | `lora.py::load_minimax_h3_turbo_lora` → pipeline hook/sampling validation → `DiffusionLoRAManager` | | FlashGen native LoRA、grouped QKV、adapter sigma schedule、legacy manager binding | [MMH3-2m](rules-cache-task.md#mmh3-2m-flashgen-native-lora-必须保持-artifactschedule-与-legacy-manager-边界) | `npu/lora.py::load_minimax_h3_native_lora` → H3 pipeline active-adapter validation → unchanged `DiffusionLoRAManager` ^[PR #6666] | +| FastH3、fastvideo-lora-v2、four-step、load-time fusion | [MMH3-2n](rules-cache-task.md#mmh3-2n-fasth3-只接受精确-artifact-并在启动时一次性融合) | `fasth3.py` identity/strict fusion → pipeline serving/request gate | | step execution、continuous batching、multi-document packing、rank-0 prepare failure | [MMH3-2k](rules-cache-task.md#mmh3-2k-h3-step-execution-必须保持请求状态attention-文档和-rank-0-prepare-隔离) | `pipeline_minimax_h3.py::{prepare_encode,denoise_step,step_scheduler}` → `batched_packing.py` → runner | | DLO、TP-local、resident layers、allocator cache、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → `BoundedAllocatorCache`/`PinnedModuleStager` → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | From adcf7cb6f00b6c65698e0c7bb71d5e272ee2a1c2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:26:23 +0800 Subject: [PATCH 401/465] knowledge(vllm-omni): sync 2052deef (PR #6750) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 4 +++- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-output-lifecycle.md | 10 +++++++++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 24 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e4d031026..8ce82e54a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5e06d2e73c944fdfae7767b36557f4966ea2069b + previous_audited_sha: 86b85c078bc041e04aee4c4d9167fb10fb1994c7 audited_ref: main - audited_sha: 86b85c078bc041e04aee4c4d9167fb10fb1994c7 + audited_sha: 2052deef075961c9d6c4acee36e8e096d2814df3 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 31530acfb..4636f5462 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -86b85c078bc041e04aee4c4d9167fb10fb1994c7`. The canonical machine baseline +2052deef075961c9d6c4acee36e8e096d2814df3`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index f66052c27..49194835b 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -121,7 +121,9 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie cuda and L4' --run-level full_model`; its two GGUF cases cover Z-Image-Turbo and FLUX.2-klein. PR evidence reports local loading/registration recovery, while the L4 full-model run was deferred to Buildkite, so do not record a full test pass from - this change alone. ^[PR #6303 / issue #6022] + this change alone. A parent-side GGUF import/pin is insufficient for spawned stages: the child + must reload general plugins before engine construction; this is not post-change full-model proof. + ^[PR #6303 / issue #6022] ^[PR #6750] ## NPU diffusion-attention selector regression boundary diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2fbb7058c..b2cd127a4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 86b85c07` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2052deef` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 26e471baa..87428a8d5 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 86b85c07` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 2052deef` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index a289710cb..3a2000cf8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py] +sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py] confidence: high --- @@ -119,6 +119,14 @@ confidence: high - 触发：修改 diffusion result pump、共享 `_rpc_futures`/`_output_futures`、`asyncio.wait_for()` 超时、请求 abort 或 shutdown future cleanup。\n- 强制：共享 Future 在 `done()` 检查后仍必须通过 `try_set_result`/`try_set_exception` 解析，并在 helper 内捕获 `concurrent.futures.InvalidStateError`，丢弃取消或已完成 Future 的晚到结果/异常，使 singleton pump 继续运行；仅 freshly-created 且尚未共享的本地 Future 可直接解析。\n- 禁止：把 `not fut.done()` 与 `set_result()`/`set_exception()` 当作原子操作；在共享 Future 上保留未保护的 resolve 调用；让取消竞态异常逃逸并杀死 result pump，随后仍以健康检查通过推断任务会完成。\n- 验收：用 `done()` 固定返回 false 但实际已取消的 racy Future 覆盖 `COMPUTE_DONE`、`OUTPUT_READY`、batch split 与 shutdown cleanup，断言 pump 不抛 `InvalidStateError`、取消 Future 保持取消、同批健康 Future 仍完成，并回归相邻并发测试。\n^[PR #5983] +## DIFF-1ab — spawned StageDiffusionProc 必须在 engine 前重载通用插件 + +- 触发：修改 `StageDiffusionProc.run_diffusion_proc`、spawned diffusion engine startup 或 general plugin loader。 +- 强制：child 先安装 parent-death signal 与 SIGTERM/SIGINT handlers，再 `load_omni_general_plugins()`，最后构造 engine/proc；spawn fresh interpreter 不继承 parent plugin side effects。 +- 禁止：在 parent import/pin 后假定 child 已注册 loader hook，或将此 ordering 外推为任意 plugin 或完整模型 startup 成功。 +- 验收：mock parent-death signal、SIGTERM/SIGINT handlers、plugin reload 与 construction 的精确顺序； + plugin failure 必须在 engine construction 前原样传播。GGUF child reload 仍须独立 full-model proof。^[PR #6750] + ## DIFF-1s — diffusion output type 必须从模型声明闭合到 topology 与 formatter - 触发：diffusion pipeline 产生 image/video/audio/action 等输出，或修改 model metadata、registry alias、topology `final_output_type`、formatter/post-process 与 multiprocess 传递路径。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index eda2fe283..dbbda8c1d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 86b85c07` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2052deef` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 4fefb8a07..6072a2362 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 86b85c07` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2052deef` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e3c2fa53e..418f4a519 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 86b85c07` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2052deef` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 26863f7af..61dacc102 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 86b85c07` 验证存在 +- 源码校验：以上路径均已在 `main @ 2052deef` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index f50d63c7f..d3ad1672f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 86b85c07`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2052deef`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 5e06d2e7 \ - --to 86b85c07 \ + --from 86b85c07 \ + --to 2052deef \ --repo \ --mode report-only ``` From 1443dfd5136f6df3187ff1e975ee7b0362fda0af Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:30:20 +0800 Subject: [PATCH 402/465] knowledge(vllm-omni): sync b8643740 (PR #6803) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-loader-contract.md | 20 +++++++++++++++++-- .../components/model-executor/rules.md | 4 ++-- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/mimo-audio/_index.md | 8 ++++++-- .../models/mimo-audio/architecture.md | 7 +++++-- 13 files changed, 43 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8ce82e54a..d03fef88f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 86b85c078bc041e04aee4c4d9167fb10fb1994c7 + previous_audited_sha: 2052deef075961c9d6c4acee36e8e096d2814df3 audited_ref: main - audited_sha: 2052deef075961c9d6c4acee36e8e096d2814df3 + audited_sha: b864374089749ad3b4e6d01cb5d4f627beeb6174 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4636f5462..4fdef7499 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2052deef075961c9d6c4acee36e8e096d2814df3`. The canonical machine baseline +b864374089749ad3b4e6d01cb5d4f627beeb6174`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b2cd127a4..b01a9efa9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2052deef` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b8643740` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 87428a8d5..7842ab6ac 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2052deef` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ b8643740` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index dbbda8c1d..12bd0ea03 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2052deef` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b8643740` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6072a2362..be11833e4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2052deef` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b8643740` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md index 8564507c6..3bf0435ca 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-loader-contract.md @@ -4,13 +4,13 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py", "PR #6138", "PR #6640", vllm_omni/engine/stage_init_utils.py, tests/engine/test_async_omni_engine_stage_init.py, tests/engine/test_stage_engine_args.py, "PR #6830", vllm_omni/engine/arg_utils.py, tests/engine/test_arg_utils.py] +sources: ["PR #4730", "PR #4765", "PR #4958", "PR #5777", "PR #5824", vllm_omni/model_executor/model_loader/, "PR #5910", "PR #6119", "PR #5791", "vllm_omni/model_executor/models/common/qwen3_code_predictor.py", "vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py", "PR #6138", "PR #6640", vllm_omni/engine/stage_init_utils.py, tests/engine/test_async_omni_engine_stage_init.py, tests/engine/test_stage_engine_args.py, "PR #6830", vllm_omni/engine/arg_utils.py, tests/engine/test_arg_utils.py, "PR #6803", vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py, vllm_omni/model_executor/models/mimo_audio/mimo_audio.py] confidence: high --- # loader 合同 -`loader-contract` 审查组的 `EXEC-2a`–`EXEC-2c`：dtype 与 config 获取、fused shard 完整性、多模块 checkpoint 载入。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 +`loader-contract` 审查组的 `EXEC-2a`–`EXEC-2i`：dtype 与 config 获取、fused shard 完整性、多模块 checkpoint 载入与 wrapper runtime callable 委托。触发条件与其余审查组见 [model-executor 共享规则](rules.md) 的 Direct 代码快速入口。 ## EXEC-2a — loader 的 dtype 与 config 获取必须显式、最小化 @@ -98,3 +98,19 @@ confidence: high success 还须断言 resolved tokenizer path 被赋值且 warning logger 未调用。模型专有 root completeness 另由 owner 验证。unit fixture 不证明真实 checkpoint load、音质、GPU E2E 或跨平台支持。 ^[PR #6640] ^[PR #6830] + +## EXEC-2i — concrete `SupportsPP` wrapper 必须显式委托 required callable + +- 触发：上游 vLLM 的 `SupportsPP` interface/protocol 漂移，或包含 nested vLLM model 的 concrete + wrapper 声明/实现该 interface。 +- 强制：concrete consumer 必须在其自身运行时对象上显式提供 required callable；MiMo fused + thinker/talker 在构造 nested `Qwen2ForCausalLM` 后，必须将 + `self.make_empty_intermediate_tensors` 委托为 + `self.model.make_empty_intermediate_tensors`。 +- 禁止：把 `Protocol` 中的 bare annotation 当作会在 runtime 创建属性；把这一个 MiMo wrapper + 修复泛化为所有 `SupportsPP` consumer 已实现委托，或以此宣称 PP>1 已验证。 +- 验收：MiMo stage-0 construct/load 路径必须能取得该 callable 并完成其已覆盖的初始化。当前 + 证据仅为修复前的 `AttributeError` 与 A6000 上两 stage `LOAD OK`；没有新增 unit 或 PP>1 + 覆盖。审计中 `MiMoAudioToken2WavForConditionalGenerationVLLM` 与 + `CovoAudioCode2WavForConditionalGeneration` 的同类缺口仍须由各自 owner 单独确认。 + ^[PR #6803] diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 9e4907f68..731f979e7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -24,7 +24,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | cross-stage embedding width、pre-projection buffer | `bridge-batch`：`EXEC-1d` | stage HF config → `get_inputs_embeds_size()` → `GPUARModelRunner.inputs_embeds` → model projection | | loader dtype、只取 checkpoint config、避免整仓权重下载 | `loader-contract`：`EXEC-2a` | `vllm_omni/model_executor/model_loader/weight_utils.py::download_weights_from_hf_specific` → `vllm_omni/model_executor/models/<命中模型>` loader | | stage `model_subdir`/`tokenizer_subdir`、partial snapshot、revision/cache repair | `loader-contract`：`EXEC-2h` | `engine/stage_init_utils.py` → stage engine args → model-specific root resolver | -| fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b` | H3 text encoder fused owner；其他模型先确认目标 main 是否已有 fused parameter | +| fused projection、HF source shard 完整性、consumer 委托、packed TP | `loader-contract`：`EXEC-2b`, `EXEC-2i` | H3 text encoder fused owner；nested wrapper 的 required callable 必须在 concrete runtime object 上委托 | | async Omni output、background builder、D2H snapshot、connector drain/fallback | `async-output`：`EXEC-5a` | `worker/gpu_ar_model_runner.py::{_should_use_async_omni_output,OmniAsyncGPUModelRunnerOutput}` → platform runner → connector output | | upstream runner/output API drift、`num_nans`、sample-budget 或 quant loader helper moved | `async-output` + output-contract | live upstream runner/output dataclasses → `gpu_{ar,generation}_model_runner.py` / `outputs/output_processor.py` → diffusion quant configs；moved symbol 必须用目标 release 的 canonical import，而非保留旧 fallback ^[PR #6606] | | NPU ngram speculative decode、`SchedulerOutput` mutable map、profiling timing 或 upstream NPU runner ownership | `platform-backends`：`EXEC-10e`, `EXEC-13h` | `platforms/npu/worker/{npu_ar_model_runner,npu_generation_model_runner,npu_model_runner}.py` → inherited invalid-draft trimming / profiler gate → upstream vLLM-Ascend owner | @@ -41,7 +41,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `connector-capability` | full-payload/connector stage 的 worker runner validation | `EXEC-3c` | | `stage-runtime` | stage 数、replica、inline/subprocess 与设备容量 | `EXEC-3d` + 本页 stage 并行度验收 | | `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`–`EXEC-1i`，见 [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | -| `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装、stage snapshot/subdir | `EXEC-2a`–`EXEC-2h`，见 [loader 合同](rules-loader-contract.md) | +| `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装、stage snapshot/subdir、wrapper callable 委托 | `EXEC-2a`–`EXEC-2i`，见 [loader 合同](rules-loader-contract.md) | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a`, `EXEC-5b` | | `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | | `image-task-envelope` | shared image/video task example、video output consumer 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6d`，见 [image task envelope 合同](rules-image-task-envelope.md) | diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 418f4a519..065f2167a 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2052deef` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b8643740` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 61dacc102..032f89617 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2052deef` 验证存在 +- 源码校验：以上路径均已在 `main @ b8643740` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d3ad1672f..d47190677 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2052deef`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b8643740`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 86b85c07 \ - --to 2052deef \ + --from 2052deef \ + --to b8643740 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/mimo-audio/_index.md b/knowledge/repos/vllm-omni/models/mimo-audio/_index.md index 01a109686..a9b98cca5 100644 --- a/knowledge/repos/vllm-omni/models/mimo-audio/_index.md +++ b/knowledge/repos/vllm-omni/models/mimo-audio/_index.md @@ -1,10 +1,10 @@ --- title: "MiMo-Audio（融合 thinker+talker 单 AR stage 语音模型）" created: 2026-07-21 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: ["PR #6559", recipes/XiaomiMiMo/MiMo-Audio.md, vllm_omni/model_executor/models/mimo_audio/, vllm_omni/deploy/mimo_audio.yaml, vllm_omni/deploy/mimo_audio_5090d.yaml, vllm_omni/model_executor/stage_input_processors/mimo_audio.py] +sources: ["PR #6559", "PR #6803", recipes/XiaomiMiMo/MiMo-Audio.md, vllm_omni/model_executor/models/mimo_audio/, vllm_omni/deploy/mimo_audio.yaml, vllm_omni/deploy/mimo_audio_5090d.yaml, vllm_omni/model_executor/stage_input_processors/mimo_audio.py] --- # MiMo-Audio @@ -36,6 +36,10 @@ sources: ["PR #6559", recipes/XiaomiMiMo/MiMo-Audio.md, vllm_omni/model_executor `vllm_omni/model_executor/stage_input_processors/mimo_audio.py`。 - 依赖共享模块：vLLM qwen2_audio 处理栈、SharedMemoryConnector、 [Config 组件](../../components/configuration/architecture.md)。 +- vLLM 0.28 的 `SupportsPP` bare annotation 不会为 wrapper 创建 runtime attribute；fused + thinker/talker 在构造 nested Qwen2 后显式委托 + `make_empty_intermediate_tensors`。这只覆盖观测到的 stage-0 load：Token2Wav 与 CovoAudio + 仍是需各自确认的缺口，不能外推为通用 `SupportsPP` 或 PP>1 支持。^[PR #6803] ## 目录内容 diff --git a/knowledge/repos/vllm-omni/models/mimo-audio/architecture.md b/knowledge/repos/vllm-omni/models/mimo-audio/architecture.md index 3cd374b75..df63801c4 100644 --- a/knowledge/repos/vllm-omni/models/mimo-audio/architecture.md +++ b/knowledge/repos/vllm-omni/models/mimo-audio/architecture.md @@ -1,10 +1,10 @@ --- title: "MiMo-Audio 架构" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/mimo_audio/mimo_audio.py, vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py, vllm_omni/model_executor/models/mimo_audio/mimo_audio_code2wav.py, vllm_omni/model_executor/models/mimo_audio/cuda_graph_decoder_wrapper.py, vllm_omni/model_executor/models/mimo_audio/modeling_audio_tokenizer.py, vllm_omni/model_executor/models/mimo_audio/quantization.py, vllm_omni/model_executor/models/mimo_audio/pipeline.py, vllm_omni/model_executor/stage_input_processors/mimo_audio.py, vllm_omni/model_executor/models/mimo_audio/config_mimo_audio.py] +sources: ["PR #6803", vllm_omni/model_executor/models/mimo_audio/mimo_audio.py, vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py, vllm_omni/model_executor/models/mimo_audio/mimo_audio_code2wav.py, vllm_omni/model_executor/models/mimo_audio/cuda_graph_decoder_wrapper.py, vllm_omni/model_executor/models/mimo_audio/modeling_audio_tokenizer.py, vllm_omni/model_executor/models/mimo_audio/quantization.py, vllm_omni/model_executor/models/mimo_audio/pipeline.py, vllm_omni/model_executor/stage_input_processors/mimo_audio.py, vllm_omni/model_executor/models/mimo_audio/config_mimo_audio.py] --- # MiMo-Audio 架构 @@ -18,6 +18,9 @@ sources: [vllm_omni/model_executor/models/mimo_audio/mimo_audio.py, vllm_omni/mo thinker/talker 分离);Qwen2 backbone + CUDA-graph 化的 local transformer （`MiMoLocalDecodeCudaGraph` 等,逐步产 RVQ 码）;自有采样器 `MiMoSampler`/`MiMoLocalSamplerTensor`。 +- vLLM 0.28 只在 `SupportsPP` protocol 标注 callable，不会在 runtime 提供它；因此 stage-0 + wrapper 在 nested Qwen2 构造后委托 `make_empty_intermediate_tensors`。这不是 Token2Wav、Covo + 或 PP>1 的实现/验证声明。^[PR #6803] - 专有 stage 1（`mimo_audio_code2wav.py` + vendored `modeling_audio_tokenizer.py`/`quantization.py`）：VQ 反量化 → AudioDecoder → Vocos 声码器;`CUDAGraphMiMoDecoderWrapper` 把**整条解码路径**按 From f2a37e4896bc1638cd7de813b6146ca6b4ab785f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:32:19 +0800 Subject: [PATCH 403/465] knowledge(vllm-omni): sync 58cb8de6 (PR #6827) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d03fef88f..303686972 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2052deef075961c9d6c4acee36e8e096d2814df3 + previous_audited_sha: b864374089749ad3b4e6d01cb5d4f627beeb6174 audited_ref: main - audited_sha: b864374089749ad3b4e6d01cb5d4f627beeb6174 + audited_sha: 58cb8de68bfef2993440b568907db60f448956b6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4fdef7499..7a632ef41 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b864374089749ad3b4e6d01cb5d4f627beeb6174`. The canonical machine baseline +58cb8de68bfef2993440b568907db60f448956b6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b01a9efa9..4b7f9e584 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b8643740` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 58cb8de6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7842ab6ac..85fd6d62e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b8643740` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 58cb8de6` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 12bd0ea03..6366fe38c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b8643740` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 58cb8de6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index be11833e4..ccf92f932 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b8643740` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 58cb8de6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 065f2167a..e219c47d8 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b8643740` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 58cb8de6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 032f89617..d4028aeba 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b8643740` 验证存在 +- 源码校验：以上路径均已在 `main @ 58cb8de6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d47190677..edca0fe33 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b8643740`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 58cb8de6`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 2052deef \ - --to b8643740 \ + --from b8643740 \ + --to 58cb8de6 \ --repo \ --mode report-only ``` From b964cad44e3e43d2b28fefbd07ba9f52b61abef9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:34:30 +0800 Subject: [PATCH 404/465] knowledge(vllm-omni): sync 6f16f954 (PR #6814) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-system-runtime.md | 8 ++++---- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 303686972..6e6934aad 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b864374089749ad3b4e6d01cb5d4f627beeb6174 + previous_audited_sha: 58cb8de68bfef2993440b568907db60f448956b6 audited_ref: main - audited_sha: 58cb8de68bfef2993440b568907db60f448956b6 + audited_sha: 6f16f9546d18fa5094115d13502b885c761c93b8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7a632ef41..1f432df74 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -58cb8de68bfef2993440b568907db60f448956b6`. The canonical machine baseline +6f16f9546d18fa5094115d13502b885c761c93b8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4b7f9e584..44fcd3c9c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 58cb8de6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 6f16f954` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 85fd6d62e..bfdb81f1d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 58cb8de6` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 6f16f954` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index db085219e..76f519a8f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/distributed/test_a2a_permute.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", "PR #6814", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/distributed/csrc/a2a_permute.cu, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/distributed/test_a2a_permute.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] confidence: high --- @@ -83,9 +83,9 @@ confidence: high ## DIFF-4x — SymmMem Ulysses A2A 只能显式启用并保持 workspace 生命周期闭合 - 触发：修改 `ulysses_a2a_permute`、strict Ulysses all-to-all、NCCL SymmetricMemory JIT extension、CUDA graph capture 或 diffusion worker shutdown。 -- 强制：默认关闭；只在 `ulysses_degree > 1`、strict Ulysses 的正向 `(scatter,gather)=(2,1)` 与反向 `(1,2)` layout 启用，并在 worker/model 初始化时 JIT build/capability failure 直接 fail-at-init。advanced-UAA 保持其 `_ulysses_all_to_all_any_*` path；AllGather 不选择 Ulysses；strict Ulysses+Ring 的 Ulysses leg 仍可使用该 path。每个 `(device, process-group)` 只有一个单 CUDA stream 的 grow-only byte workspace；首次分配及增长必须是 collective、在增长前同步，容量内以 typed byte-slice view 复用；graph capture 中禁止增长，须先 warm up 最大 shape；worker shutdown 同步并释放 workspace。^[PR #6714] -- 禁止：把 flag 当作默认 transport、静默 fallback；按历史 shape/dtype 无界缓存 allocation；跨 stream 重用 staging buffer；capture 中隐式分配/扩容；把 CPU/mock workspace 测试或 PR benchmark 写成数学/输出 parity、普遍速度或跨硬件收益。 -- 验收：覆盖 CLI/deploy/default-stage 的显式 flag 透传与默认 false、eligible strict path 的 init build、UAA/AllGather/noneligible layout 不调用该 path、strict Ulysses+Ring 的 eligible Ulysses leg、peak workspace 复用/collective growth、capture growth fail、single-stream reject 与 shutdown release；真实多 rank hardware 仍须对 regular Ulysses 做 Q/K/V 与 reverse-output parity，再单独记录精确 shape/stack/预热后的性能。^[PR #6340] +- 强制：默认关闭；只在 `ulysses_degree > 1`、strict Ulysses 的正向 `(scatter,gather)=(2,1)` 与反向 `(1,2)` layout 启用，并在 worker/model 初始化时 JIT build/capability failure 直接 fail-at-init。advanced-UAA 保持其 `_ulysses_all_to_all_any_*` path；AllGather 不选择 Ulysses；strict Ulysses+Ring 的 Ulysses leg 仍可使用该 path。每个 `(device, process-group)` 只有一个单 CUDA stream 的 grow-only byte workspace；首次分配及增长必须是 collective、在增长前同步，容量内以 typed byte-slice view 复用；graph capture 中禁止增长，须先 warm up 最大 shape；worker shutdown 同步并释放 workspace。Fast Ulysses QKV staging 只有 input/output 同一 CUDA device、同一 dtype、input rank-3、shape 相同、output contiguous，且 inner row `stride(2)==1`、`stride(1)==size(2)`、outer stride 不小于一整 row（无 overlap）时，才可在 `CUDAGuard(input.device())` 与 current CUDA stream 上用 `cudaMemcpy2DAsync`。^[PR #6714] ^[PR #6814] +- 禁止：把 flag 当作默认 transport、静默 fallback；按历史 shape/dtype 无界缓存 allocation；跨 stream 重用 staging buffer；capture 中隐式分配/扩容；把任意 strided tensor 当作 copy-engine eligible，或把 CPU/mock workspace 测试、PR benchmark 写成数学/输出 parity、普遍速度或跨硬件收益。 +- 验收：覆盖 CLI/deploy/default-stage 的显式 flag 透传与默认 false、eligible strict path 的 init build、UAA/AllGather/noneligible layout 不调用该 path、strict Ulysses+Ring 的 eligible Ulysses leg、peak workspace 复用/collective growth、capture growth fail、single-stream reject 与 shutdown release；copy op 必须拒绝 device/dtype/rank/shape/output-contiguity/inner-row/overlap 不匹配。当前没有新增 upstream unit；仅 B300 上 row-strided BF16 bitwise copy、四-rank forward/reverse parity 和四-GPU H3 smoke 证据，不能外推为一般硬件或性能结论。^[PR #6340] ^[PR #6814] ## DIFF-4g — Ring GQA 只在通信后扩展，Hybrid Ulysses 不把 padding 当真 head diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 6366fe38c..0518b3773 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 58cb8de6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 6f16f954` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ccf92f932..fc324b735 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 58cb8de6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 6f16f954` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e219c47d8..5f9d88b23 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 58cb8de6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 6f16f954` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index d4028aeba..9668bf85f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 58cb8de6` 验证存在 +- 源码校验：以上路径均已在 `main @ 6f16f954` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index edca0fe33..009c34c76 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 58cb8de6`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 6f16f954`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from b8643740 \ - --to 58cb8de6 \ + --from 58cb8de6 \ + --to 6f16f954 \ --repo \ --mode report-only ``` From afd59573c9a1a1295fd33d55dab30b84b16656f1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:41:27 +0800 Subject: [PATCH 405/465] knowledge(vllm-omni): sync fe4a2a74 (PR #6563) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/paged-kv-control-plane.md | 37 ++++++++++--------- .../components/diffusion/rules-attention.md | 4 +- .../diffusion/rules-system-runtime.md | 13 ++++--- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-platform-backends.md | 6 +-- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../vllm-omni/models/hunyuan-image3/_index.md | 6 +-- .../models/hunyuan-image3/architecture.md | 8 +++- .../vllm-omni/models/hunyuan-image3/rules.md | 12 ++++-- 16 files changed, 61 insertions(+), 49 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6e6934aad..acbd94007 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 58cb8de68bfef2993440b568907db60f448956b6 + previous_audited_sha: 6f16f9546d18fa5094115d13502b885c761c93b8 audited_ref: main - audited_sha: 6f16f9546d18fa5094115d13502b885c761c93b8 + audited_sha: fe4a2a7477da686d12dc20c1ab70fc266e90eb35 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 1f432df74..cfefc911b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -6f16f9546d18fa5094115d13502b885c761c93b8`. The canonical machine baseline +fe4a2a7477da686d12dc20c1ab70fc266e90eb35`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 44fcd3c9c..6cbaaec4c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 6f16f954` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fe4a2a74` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index bfdb81f1d..2dd1a37de 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 6f16f954` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ fe4a2a74` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md b/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md index 754a63839..c5382761d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md +++ b/knowledge/repos/vllm-omni/components/diffusion/paged-kv-control-plane.md @@ -1,10 +1,10 @@ --- title: "Diffusion paged KV control plane" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion, scheduler] -sources: ["PR #5541", "PR #5550", vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/diffusion_kv/config.py, vllm_omni/diffusion/diffusion_kv/metadata.py, vllm_omni/diffusion/diffusion_kv/request.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py, vllm_omni/diffusion/request.py, vllm_omni/diffusion/sched/base_scheduler.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/worker/utils.py, tests/config/test_omni_config.py, tests/diffusion/diffusion_kv/test_config.py, tests/diffusion/diffusion_kv/test_metadata.py, tests/diffusion/diffusion_kv/test_request.py, tests/diffusion/diffusion_kv/test_worker_contract.py, tests/diffusion/models/hunyuan_image3/test_diffusion_kv_request.py, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/test_diffusion_config_propagation.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_model_runner.py, tests/diffusion/test_diffusion_scheduler.py] +sources: ["PR #5541", "PR #5550", "PR #6563", vllm_omni/config/omni_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/diffusion_kv/, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py, vllm_omni/diffusion/request.py, vllm_omni/diffusion/sched/, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/diffusion_kv/, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/test_diffusion_model_runner.py] --- # Diffusion paged KV control plane @@ -47,7 +47,10 @@ sources: ["PR #5541", "PR #5550", vllm_omni/config/omni_config.py, vllm_omni/dif `[prefix | target | suffix]`，`prefix_len` 可复用，`target_len` 每步重写，`seq_len` 是首步完整 allocation boundary，并允许 `prefix_len + target_len < seq_len`。Hunyuan 分别从 `gen_timestep_scatter_index[row,-1]`、生成图+timestep/guidance token 和 `real_pos[row,-1]` - 取得三段长度；prompt/reference image 已在同一 self-attention sequence，所以 `kv_contexts=()`。 + 取得三段长度；prompt/reference image 已在同一 self-attention sequence，所以 `kv_contexts=()`。Scheduler + 只持有 logical allocation/generation；Worker 才持有 physical KV tensors、native BlockTables、slot mapping + 与 attention metadata。first prefill 的 query/sequence span 必须覆盖整个 logical allocation；后续 denoise + 仅以 `prefix_len` 为 `kv_start_pos` 写入 target span。 - 禁止：把独立 cross/joint context 塞入主 token axis，或把可变 `DiffusionKVRequest` 送给 Worker； admission 必须把 tuple 移入 `SchedulerRequestState` 并清空 Worker-bound 字段。 - 验收：CFG=1/2、suffix、reference image 与独立 context 都验证 identity/length/owner。失败路径也要 @@ -79,23 +82,21 @@ sources: ["PR #5541", "PR #5550", vllm_omni/config/omni_config.py, vllm_omni/dif 追加第四个 RPC 位置参数，保持 dense 历史三参数 shape；batch/step 则随完整 `DiffusionSchedulerOutput` 传输。`prepared_layout` 仍是独立的 model-owned payload，不能从它派生 block ID。Worker wrapper 也只在非空时向旧 runner 增加 keyword，避免破坏 dense/custom runner。 -- 强制：W0 尚无 allocator/installer，因此 `paged_scheduler` 接受 metadata 但不得强制要求； - `dense_legacy` 必须拒绝 metadata。带 metadata 的多请求 wave 不能走 DLO 的 list-request 快捷 RPC， - 因为该路径没有逐请求 snapshot 参数；应退回逐请求 dispatch。^[PR #5550] -- 禁止：把这些可变 dataclass 当成已验证的安全边界。当前只验证 mode 与顶层 request identity， - 不检查 generation/sequence/block ID 正数、`prefix+target<=seq_len`、精确 sequence/context 集合、 - cache-group 数/顺序、block-table 长度/范围或 null block。上述检查必须由未来 Scheduler producer - 对照 `SchedulerRequestState.diffusion_kv_requests`，再由 Worker installer 对照 rank-local native - `KVCacheConfig` 与 generation registry，而不是用 DTO 自证正确。 +- 强制：`paged_scheduler` 的 request envelope 必须携带并消费 metadata，Worker 安装 physical cache/ + BlockTables/slot mapping 后才能激活 paged attention；`dense_legacy` 必须拒绝 metadata。带 metadata 的 + 多请求 wave 不能走没有逐请求 snapshot 参数的 DLO list-request 快捷 RPC。^[PR #5550] ^[PR #6563] +- 禁止：把这些可变 dataclass 本身当成安全边界。Worker installer 必须对照 rank-local native + `KVCacheConfig` 与 generation registry，验证非空 request、non-negative generation/sequence ID、唯一 + sequence/context identity、cache-group 数、按 token length 推导的 block 数、row capacity 和 block ID + type/range；stale/conflicting generation 必须拒绝，重复同 snapshot/generation 幂等。`prefix+target<=seq_len` + 与 sequence 引用的 context 集合仍由 Scheduler producer/model layout owner 保证，不能从 DTO 自证。 - 验收：request/batch/step 覆盖 metadata 缺省、传播与三方 ID mismatch；dense metadata fail-fast， DLO metadata wave 不进入 list-request 分支，legacy positional shape 与 `prepared_layout` identity - 保持。严格 layout/range/generation 测试只能在独立 truth owner 落地后升级为已实现合同。 + 保持；installer 覆盖 invalid group/count/range/capacity、duplicate identity、stale/conflicting generation、 + idempotent replay，以及 append/apply failure rollback。^[PR #6563] ## 证据边界 -- 目标实现只提供 logical request/control-plane contracts、request-scoped Scheduler→Worker metadata - DTO/RPC 与 native-manager conformance tests；没有 cache manager、physical allocation、Worker - installer/registry、BlockTable、slot mapping 或 paged attention。metadata 可选且未被消费，内部 - 数值字段也未做严格校验，因此不能把“到达 runner”解释为 allocation 可用。 -- 目标 Hunyuan E2E 运行 `dense_legacy`，只证明既有 dense path；不能作为 Scheduler-managed paged - data plane、attention 或 cross-request caching 正确性的证据。 +- HunyuanImage3 现已在 NVIDIA GPU 与 Ascend NPU 的 `request_execution` 路径消费 Scheduler-paged + metadata；它不证明 `denoise_step`、continuous batching/多请求 admission、导入 AR-to-DiT KV、Ring SP、 + AllGather-KV 或跨请求 prefix reuse。空 hash 仍意味着 request-local allocation，而非 prefix hit。^[PR #6563] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index b6f449f84..b9bd2fe1d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] +sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] confidence: high --- @@ -138,7 +138,7 @@ confidence: high ## DIFF-1z — 混合 causal/full attention 必须经过共享 Attention 能力路径 - 触发：diffusion 模型包含文本 causal 行与 image/timestep full-attention 行的混合序列，或修改 `full_attn_spans`、attention mask 与高分辨率长序列 dispatch。 -- 强制：通过 `vllm_omni.diffusion.attention.layer.Attention` 传递 `AttentionMetadata`；piecewise backend 消费连续的 `full_attn_spans`，不支持时才使用按请求构造并复用的 dense mask，且 mask 形状必须覆盖完整 text+image 序列。 +- 强制：通过 `vllm_omni.diffusion.attention.layer.Attention` 传递 `AttentionMetadata`；piecewise backend 消费连续的 `full_attn_spans`，不支持时才使用按请求构造并复用的 dense mask，且 mask 形状必须覆盖完整 text+image 序列。strict Ulysses paged path 必须保存/恢复 SP padding 并要求 `query_lens`、`image_mask`、`position_ids`；Ring、AllGather 和 non-strict Ulysses 必须 fail-closed，不能借 paged marker 落入其他 path。^[PR #6563] - 禁止：在模型内直接调用 raw `scaled_dot_product_attention`，绕过 diffusion attention backend/configuration；把所有 full rows 当作任意离散 spans；在每个 denoise step 重建只依赖 `token_types` 的 SxS mask，或因 fallback 方便而关闭 backend 能力检查。 - 验收：CPU/mock 覆盖 contiguous full-span、piecewise path、dense fallback、batch mask shape 与 piecewise/dense 输出一致性；高分辨率请求另验证实际 backend 选择、显存边界和 attention mask 不被重复构造。^[PR #5194] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index 76f519a8f..7b39b26ec 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", "PR #6814", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/distributed/csrc/a2a_permute.cu, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/distributed/test_a2a_permute.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", "PR #6814", "PR #6563", vllm_omni/diffusion/attention/, vllm_omni/diffusion/diffusion_kv/, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/npu/platform.py, tests/diffusion/diffusion_kv/, tests/diffusion/attention/test_piecewise_attn.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] confidence: high --- @@ -64,7 +64,8 @@ confidence: high - 触发：启用 paged scheduler、增加 diffusion attention backend 或平台 override。 - 强制：启动阶段解析并验证每层实际 backend 与当前平台；走 Omni platform hook，保留 - piecewise/SP 等能力。尚无生产 caller 的模式必须明确告警并文档化。 + piecewise/SP 等能力。HunyuanImage3 paged requests 已有 GPU/NPU production caller；NPU SDPA 只可作 + startup memory-profile 的 dense fallback，正式 paged request 必须走 native backend。 - 禁止：未知平台继承 GPU 默认；分配 cache 后到首次 dispatch 才失败；静默占用 paged KV 却继续 dense；直接换用 upstream backend 丢失 Omni 能力。 - 验收：unsupported platform/backend fail-fast；CUDA/NPU 对应 lane 和正式模型 E2E 证明 @@ -173,9 +174,9 @@ confidence: high ## DIFF-4p — Scheduler-managed Diffusion KV 必须原子分配并绑定请求生命周期 - 触发：启用 `paged_scheduler`，或修改 Worker KV geometry discovery、显存 profile、Scheduler admission、CFG allocation、`NewRequestData`/DLO RPC envelope 和 request terminal cleanup。 -- 强制：模型加载后从每个 Worker 的 cache-enabled attention 生成 rank-local native `FullAttentionSpec`，以最大 per-rank profile batch 测量 KV headroom，再通过 `get_kv_cache_configs()`、`generate_scheduler_kv_cache_config()` 和 `resolve_kv_cache_block_sizes()` 构造 Worker/Scheduler 配置。Scheduler 侧只能以 native `KVCacheManager` 的薄 wrapper 为物理 owner；每个 CFG sequence 按完整首步 `seq_len` 以 `full_sequence_must_fit=True` 分配，先按空池所需 block 总数做 never-fit 检查，并在所有 sequence 成功前保持原子性。生成的 `DiffusionKVMetadata` 必须与请求绑定并随完整 `NewRequestData` envelope 传到 Worker；完成、取消、错误、pop 和 close 都必须释放整组 allocation，preemption 只保留既有 allocation。 +- 强制：模型加载后从每个 Worker 的 cache-enabled attention 生成 rank-local native `FullAttentionSpec`，以最大 per-rank profile batch 测量 KV headroom，再通过 `get_kv_cache_configs()`、`generate_scheduler_kv_cache_config()` 和 `resolve_kv_cache_block_sizes()` 构造 Worker/Scheduler 配置。Scheduler 是 logical allocation/generation owner，Worker 是 physical tensors/BlockTables/slot mappings owner；每个 CFG sequence 按完整首步 `seq_len` 以 `full_sequence_must_fit=True` 分配，先按空池所需 block 总数做 never-fit 检查，并在所有 sequence 成功前保持原子性。生成的 `DiffusionKVMetadata` 必须与请求绑定并随完整 `NewRequestData` envelope 传到 Worker；完成、取消、错误、pop 和 close 都必须释放整组 allocation，preemption 只保留既有 allocation。^[PR #6563] - 禁止：引入独立 block pool、KV spec、refcount 或 physical lifecycle；把部分 CFG 分配当作成功；把临时容量不足误报为永久失败，或让 never-fit 请求在 FIFO 头部无限阻塞；在 DLO DP wave 中拆开 request 与对应 metadata，或让 scheduler/busy-loop 异常逃逸而不唤醒请求流。 -- 验收：覆盖 dense 默认旁路、rank/spec/memory mismatch、profile 顺序、CFG partial rollback、临时压力返回 `None` 后 FIFO 重试、空池也放不下时只终止该请求、native allocation error、preemption metadata 保留，以及 finish/cancel/error/pop/close 全部释放；DLO 多 rank RPC 必须逐 envelope 保持 request 与 metadata 配对。 ^[PR #6094] +- 验收：覆盖 dense 默认旁路、rank/spec/memory mismatch、profile 顺序、CFG partial rollback、临时压力返回 `None` 后 FIFO 重试、空池也放不下时只终止该请求、native allocation error、preemption metadata 保留，以及 finish/cancel/error/pop/close 全部释放；Hunyuan first prefill 覆盖整 allocation，denoise 只写 prefix-offset target。^[PR #6094] ^[PR #6563] ## DIFF-4t — HiDream-O1 checkpoint signature 必须正向门禁 @@ -187,6 +188,6 @@ confidence: high ## DIFF-4u — Worker 物理 paged KV 必须与 Scheduler 逻辑分配分层并原子安装 - 触发：启用或修改 diffusion Worker 的 native paged KV data plane、请求 row registry、paged attention adapter、BlockTables 更新或唤醒/终止清理。 -- 强制：Scheduler 只拥有 logical block allocation、generation 和 `DiffusionKVMetadata`；每个 Worker rank 消费自己的 `KVCacheConfig`，拥有物理 KV tensors、native `BlockTables` 和 request/sequence/context rows。安装 metadata 前必须校验 group/count/range/capacity，并以 staged append、apply 和失败 rollback 原子发布；重复 generation 幂等，冲突或过期快照拒绝，BlockTables 变更使已准备的 attention batch 失效，终止时先清 Worker rows 而不释放 Scheduler blocks。 +- 强制：Scheduler 只拥有 logical block allocation、generation 和 `DiffusionKVMetadata`；每个 Worker rank 消费自己的 `KVCacheConfig`，拥有物理 KV tensors、native `BlockTables` 和 request/sequence/context rows。安装 metadata 前必须校验 group/count/range/capacity，并以 staged append、apply 和失败 rollback 原子发布；重复 generation 幂等，冲突或过期快照拒绝，BlockTables 变更使已准备的 attention batch 失效，终止时先清 Worker rows 而不释放 Scheduler blocks。request-mode runner 负责从 snapshot 构造并激活 prefill/denoise row metadata；model 只提供 Q/K/V layout 与 `full_attn_spans`。 - 禁止：Worker 自行分配或释放 Scheduler-owned logical blocks；把部分 row 安装、native append/apply 异常或 stale snapshot 当作成功；从 padding 猜异构 prefix 的逻辑布局；在清理或 wake refresh 后继续复用旧的 native metadata/buffer view。 -- 验收：CPU/mock 覆盖 rank-local config、请求/上下文 row mapping、重复/过期/冲突 generation、非法 block 与容量、append/apply rollback、幂等清理和 wake refresh；GPU 另验证非连续 BlockTables、GQA、异构 prefix 与 piecewise attention。该 PR 不证明跨 stage KV transport、W2 prefix-cache 或真实 HunyuanImage3 E2E。^[PR #6102] +- 验收：CPU/mock 覆盖 rank-local config、请求/上下文 row mapping、重复/过期/冲突 generation、非法 block 与容量、append/apply rollback、幂等清理和 wake refresh；GPU/NPU 另验证非连续 BlockTables、32Q/8KV GQA、异构 prefix 与 piecewise attention。Hunyuan paged request execution 不证明 `denoise_step`、cross-request prefix reuse、AR KV、negative CFG、Ring 或 AllGather。^[PR #6102] ^[PR #6563] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0518b3773..b8bb38f71 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 6f16f954` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fe4a2a74` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index fc324b735..f15b19abd 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 6f16f954` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fe4a2a74` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index b7f4ac462..836798696 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102", "PR #6054", "vllm_omni/platforms/npu/platform.py", tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6674", vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py] +sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102", "PR #6563", "PR #6054", vllm_omni/platforms/npu/platform.py, tests/platforms/npu/test_diffusion_platform.py, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6674", vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py] confidence: high --- @@ -80,9 +80,9 @@ confidence: high ## EXEC-13e — diffusion paged KV 必须经平台 native hook 闭合 - 触发：平台为 diffusion paged KV 增加或修改 BlockTables、native attention metadata builder，或 common adapter 需要支持 CUDA 与 NPU 的不同 native contract。 -- 强制：common `OmniPlatform` 通过 `get_diffusion_kv_block_tables_cls()` 与 `build_diffusion_kv_attn_metadata()` 提供默认 native 实现；NPU 必须返回 `AscendBlockTables`，把 `seq_lens_cpu` 转为 `seq_lens_np`，并使用 `AscendAttentionState.ChunkedPrefill`。common diffusion data plane 只调用 platform hooks，并在 layer registration 阶段验证 resolved backend 与 hook 能力。 +- 强制：common `OmniPlatform` 通过 `get_diffusion_kv_block_tables_cls()`、`build_diffusion_kv_attn_metadata()`、backend specialization 与 prewrite hook 提供默认 native 实现；NPU 必须返回 `AscendBlockTables`，把 `seq_lens_cpu` 转为 `seq_lens_np` 并使用 `AscendAttentionState.ChunkedPrefill`，在 spec discovery 前设 Ascend 支持的 block geometry，并为 FIA 在 piecewise segments 前恰好 prewrite complete K/V span。strict Ulysses 的 Ascend paged backend 必须绕开上游 PCP dispatch。common diffusion data plane 只调用 platform hooks，并在 layer registration 阶段验证 resolved backend 与 hook 能力。^[PR #6563] - 禁止：在 common adapter 直接导入 `vllm_ascend` 或硬编码 CUDA `BlockTables`/metadata 假设；把 platform capability 延迟到首次 attention dispatch；平台未提供 native contract 时静默回退 dense 或继续占用 paged KV。 -- 验收：default 与 NPU hook 的 class、metadata 参数和 backend selection 通过 mock/contract 测试，unsupported hook/backend 在物理 cache allocation 前 fail fast；CUDA 与 NPU lane 分别验证 native metadata construction，未运行真实 NPU 硬件时不得宣称 Ascend 生产 parity。^[PR #6102] +- 验收：default 与 NPU hook 的 class、metadata 参数、block geometry、prewrite 与 backend selection 通过 mock/contract 测试，unsupported hook/backend 在物理 cache allocation 前 fail fast；CUDA 与 NPU lane 分别验证 native metadata construction。硬件证据限于 PR 的 H200/910B request-mode lanes，不能推出其他平台、Ring/AllGather 或性能一般化。^[PR #6102] ^[PR #6563] ## EXEC-13f — NPU diffusion attention 必须只为会触达 MindIE-SD 的后端提前导入 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 5f9d88b23..fcb288a3c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 6f16f954` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fe4a2a74` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 9668bf85f..5f133090c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 6f16f954` 验证存在 +- 源码校验：以上路径均已在 `main @ fe4a2a74` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 009c34c76..476d0f0dc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 6f16f954`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fe4a2a74`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 58cb8de6 \ - --to 6f16f954 \ + --from 6f16f954 \ + --to fe4a2a74 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md index 7c52285fa..b40d21bfc 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md @@ -1,10 +1,10 @@ --- title: "HunyuanImage3" created: 2026-07-10 -updated: 2026-09-02 +updated: 2026-09-05 type: index tags: [vllm-omni, models, hunyuan-image3] -sources: ["PR #5541", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py] +sources: ["PR #5541", "PR #6563", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py] --- # HunyuanImage3 @@ -25,7 +25,7 @@ sources: ["PR #5541", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan | 遇到什么 | 查看哪里 | |---|---| | 修改公开入口、prompt、AR→DiT、`model_extras`、shared task examples、条件图、size 或 seed | [开发快速入口和 rules](rules.md#direct-开发快速入口) | -| Scheduler-managed paged KV、prepared layout、CFG logical prefix 或空 hash 边界 | [shared paged KV control plane](../../components/diffusion/paged-kv-control-plane.md) | +| Scheduler-managed paged KV、prepared layout、CFG logical prefix、Hunyuan Q/K/V spans 或空 hash 边界 | [shared paged KV control plane](../../components/diffusion/paged-kv-control-plane.md) | | 理解模型和 vLLM-Omni 代码地图 | [architecture](architecture.md) | | HF 接入常见偏差 | [HF alignment pitfalls](hf-alignment-pitfalls.md) | | 运行 HF baseline | [HF baseline runbook](hf-baseline-runbook.md) | diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md index 491cfff2d..382c5f674 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 核心架构" created: 2026-07-10 -updated: 2026-07-16 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models, hunyuan-image3] -sources: [vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py] +sources: ["PR #6563", vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py] --- # vLLM-Omni 核心架构 @@ -45,6 +45,10 @@ sources: [vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py, vllm ### Diffusion 侧 - Pipeline：`vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py` - DiT Transformer：`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py` +- Scheduler-paged KV：Scheduler 管理 logical allocation/generation；Worker 管理 physical pages、BlockTables + 和 metadata。模型只交 32Q/8KV 与 `full_attn_spans`，runner 激活 request-mode prefill/denoise rows：首步 + 覆盖整 allocation，之后只更新 prefix-offset target；dense prompt KV 在 paged forward 前清除。此范围不含 + `denoise_step`、AR KV、negative CFG、Ring、AllGather 或 cross-request reuse。^[PR #6563] - Tokenizer：`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py` - VAE wrapper：`vllm_omni/diffusion/models/hunyuan_image3/autoencoder.py` - MoE dispatch：`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md index 3ad025f47..9cbf69f02 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-13 updated: 2026-09-04 type: rule tags: [vllm-omni, models, hunyuan-image3] -sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", "PR #6306", "PR #6102"] +sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", "PR #6306", "PR #6102", "PR #6563", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py] --- # HunyuanImage3 开发规则 @@ -46,6 +46,7 @@ sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/dif | `size-ratio` | size=auto、ratio、batch ratio | `HY3-4a`, `HY3-4b`, `HY3-4c`, `HY3-4d`, `HY3-6g` | | `randomness` | seed、RNG、复现 | `HY3-5d`, `HY3-5e`, `HY3-5f`, `HY3-6h` | | `alignment-residual` | prompt、token、stop 已验证后仍有真实输出差异 | `HY3-3d`, `HY3-5b`, `HY3-7a`, `HY3-7b`, `HY3-7c`, `HY3-7d`, `HY3-7e` | +| `paged-kv` | Hunyuan scheduler-paged KV、Q/K/V layout、prefill/denoise span 或 mixed attention | `HY3-5h`, `HY3-5i` | ## 完整行为链 @@ -134,7 +135,7 @@ sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/dif - 触发：修改 HunyuanImage3 的 `ImageKVCacheManager`、image attention 注册、paged KV role/dtype，或为 `paged_scheduler` 调整启动 memory profile 与参考图数量。 - 强制：`ImageKVCacheManager` 必须是 `nn.Module` 并以 `forward` 暴露内部 attention，使其出现在 `named_modules()` 而不把 dense cache state 写入 `state_dict()`；内部 attention 必须显式设置 `paged_kv_cache_role="primary"` 与 `paged_kv_cache_dtype=torch.bfloat16`，该 KV dtype 独立于权重加载 dtype。paged 启动 profile 必须经过模型 owner preprocessing，使用 1024x1024、启用 CFG 和 metadata 声明的最大参考图数量，当前 HunyuanImage3 为 3 张；普通 dense warmup 仍保持独立的 512x512、无 CFG 路径。 - 禁止：保持 cache manager 为普通对象而期待 Worker spec discovery 找到嵌套 attention；把 paged KV 标记省略为默认 dense；只用一张参考图或关闭 CFG 生成 profile 后宣称容量覆盖最大请求；把 Scheduler-only `DiffusionKVRequest` 状态发送到 profile Worker execution。 - - 验收：模型测试断言 manager 是 `nn.Module`、attention identity 可从 `named_modules()` 读回且 `state_dict()` 为空；spec discovery 断言 layer、role、BF16 dtype 与 layout capability；profile 测试断言最大参考图数量、profile envelope 和 dense mode 的旁路行为。 ^[PR #6094] + - 验收：模型测试断言 manager 是 `nn.Module`、attention identity 可从 `named_modules()` 读回且 `state_dict()` 为空；spec discovery 断言 layer、role、BF16 dtype 与 layout capability；profile 后必须清除 dense `image_kv_cache_map/lens` 再 sizing/physical allocation。^[PR #6094] ^[PR #6563] - **HY3-8a — HunyuanImage3 硬件优化层必须保持平台回退、checkpoint 与 VAE 内存边界** - 触发：修改 HunyuanImage3 的 `layers/`、VAE/DiT `ResnetBlock`/`ResBlock`、patch embedding 相关块、平台分派、VAE 高分辨率内存策略或 cuDNN autotune。 @@ -146,5 +147,10 @@ sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/dif - 触发：修改 HunyuanImage3 的 `ImageKVCacheManager`、Q/K/V head layout，或在 dense 与 Scheduler-managed paged attention 之间切换 GQA 处理。 - 强制：HunyuanImage3 的 dense 路径继续把 K/V 从 8 个 KV heads 扩展到 32 个 Q heads；paged KV active 时必须保留原生压缩的 32Q/8KV 形状，由 native adapter/backend 消费 `num_heads=32` 与 `num_kv_heads=8`。 - 禁止：在 paged adapter 中把已经压缩的 K/V 再扩成 Q heads，或为兼容 generic shape 在 adapter 内猜测并折叠重复 head；也不得让 paged 分支改变既有 dense 路径的输出布局。 - - 验收：以实际 32Q/8KV geometry 覆盖 dense 与 paged 两条路径，断言 attention consumer 的 K/V shape 分别为 32 和 8，并覆盖 adapter 的 packed Q/K/V 与 native backend 调用；真实模型数值和生产 E2E 仍需独立验证。^[PR #6102] + - 验收：以实际 32Q/8KV geometry 覆盖 dense 与 paged 两条路径，断言 attention consumer 的 K/V shape 分别为 32 和 8，并覆盖 adapter 的 packed Q/K/V 与 native backend 调用；paged strict-Ulysses 还验证 padding restore 和 required `query_lens/image_mask/position_ids`。^[PR #6102] ^[PR #6563] +- **HY3-5i — paged Hunyuan 只描述模型 layout，runner 驱动请求级 execution** + - 触发：修改 HunyuanImage3 `ImageKVCacheManager`、`full_attn_spans`、Scheduler-paged prefill/denoise 或 model/runner ownership。 + - 强制：model 仅提供 32Q/8KV Q/K/V layout 与每 sequence 的 `full_attn_spans`；runner 从 Scheduler snapshot 激活 rows。first prefill 写整个 allocated sequence，后续 denoise 只从 prefix offset 写 target；paged forward 前清除 model-owned dense prompt state。 + - 禁止：model 分配 Scheduler blocks、激活 Worker runtime、复用 cross-request prefix，或在 paged path 接受 imported AR KV/negative CFG；不得把 request-mode 覆盖外推为 `denoise_step`、Ring 或 AllGather。 + - 验收：覆盖 full-allocation prefill、prefix-offset target update、32Q/8KV、full-attention spans 与 dense prompt-state clear；正式证据限 GPU/NPU request execution，其他 execution/parallel contracts另测。^[PR #6563] From 083cfc5065a94a8d3490aa5c34081680cf68bdef Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:46:30 +0800 Subject: [PATCH 406/465] knowledge(vllm-omni): sync 3d1ad0c0 (PR #6861) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 2 ++ knowledge/repos/vllm-omni/ci/rules-tts.md | 21 +++++++++++++++++++ knowledge/repos/vllm-omni/ci/rules.md | 1 + .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/qwen3-tts/rules.md | 6 +++++- 13 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 knowledge/repos/vllm-omni/ci/rules-tts.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index acbd94007..6551c83be 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 6f16f9546d18fa5094115d13502b885c761c93b8 + previous_audited_sha: fe4a2a7477da686d12dc20c1ab70fc266e90eb35 audited_ref: main - audited_sha: fe4a2a7477da686d12dc20c1ab70fc266e90eb35 + audited_sha: 3d1ad0c0f44425ad799b5d606a42d9b8a52c359c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index cfefc911b..ced255bff 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -fe4a2a7477da686d12dc20c1ab70fc266e90eb35`. The canonical machine baseline +3d1ad0c0f44425ad799b5d606a42d9b8a52c359c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 49194835b..24268c7bf 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -14,6 +14,8 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie - 处理 vLLM-Omni 的 L2/L4、模型测试配置或 CI 特有问题。 - 处理 `@hardware_test`/`hardware_marks`、SKU 与 `cards_N` 自动 marker 或按卡数切分时，先看 [测试分级与 markers](guides/test-tiers.md)，再按 OMNI-CI-1a 审核 lane 的真实收集与执行。 +- Qwen3-TTS Base/CustomVoice 的 ready/merge oracle 与 source dependency 见 + [TTS CI 规则](rules-tts.md)。 ## MiniCPM-o 4.5 full-payload regression coverage diff --git a/knowledge/repos/vllm-omni/ci/rules-tts.md b/knowledge/repos/vllm-omni/ci/rules-tts.md new file mode 100644 index 000000000..dc6a20652 --- /dev/null +++ b/knowledge/repos/vllm-omni/ci/rules-tts.md @@ -0,0 +1,21 @@ +--- +title: "vLLM-Omni TTS CI 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, ci] +sources: ["PR #6861", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, .buildkite/amd/test-amd-merge.yml, tests/e2e/online_serving/test_qwen3_tts_base.py] +confidence: high +--- + +# TTS CI 规则 + +## OMNI-CI-1d — Qwen3-TTS Base 的 dummy-ready oracle 与 real-weight merge coverage 必须分离 + +- 触发：修改 Qwen3-TTS Base/CustomVoice 的 CUDA/AMD ready 或 merge step、marker、run level、source dependencies。 +- 强制：Base online 保留 `advanced_model`，但移除 `core_model`；CUDA Ready 不收集 Base step。CUDA merge + 仍收集 Base offline+online real-weight pair，AMD merge 也保留相同 pair。新增 serving dependencies 只扩大 + CUDA merge 的 Base/CustomVoice 与 CUDA Ready 的 CustomVoice；不得外推为 AMD dependency 变更。 +- 禁止：用 Ready dummy-weight 绿灯证明 Base real-weight/EOS 行为，或把 pending merge job 写成已证明通过。 +- 验收：Ready `core_model` collect-only 对 Base 为空；CUDA/AMD merge marker matrix 仍含 Base offline+online； + source-file mutation 分别验证上述 CUDA scope。^[PR #6861] ^[issue #6855] diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 973c03f95..6ba65daf4 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -19,6 +19,7 @@ confidence: high | 新模型、nightly、CUDA/NPU、硬件 marker、镜像升级 | `OMNI-CI-1a` | `.buildkite/**` → `pyproject.toml` markers → 目标 e2e/accuracy test | | regression/guard、route census、middleware、mutation test | `OMNI-CI-1b` | 公开 app/handler → guard test；先证明旧实现会失败 | | `NIGHTLY`/`WEEKLY`/`NON_CRITICAL`、L1/L4/L5、coverage、scheduled bootstrap | `OMNI-CI-1c` | `upload_pipeline.py` → bootstrap YAML → ready/merge/nightly/weekly YAML → coverage helper | +| Qwen3-TTS Base/CustomVoice ready/merge collection 或 source dependencies | [OMNI-CI-1d](rules-tts.md#omni-ci-1d-qwen3-tts-base-的-dummy-ready-oracle-与-real-weight-merge-coverage-必须分离) | CUDA/AMD ready+merge YAML → Qwen3-TTS e2e markers | | pre-commit、SPDX、shellcheck、stability marker | `OMNI-CI-2a` | `.pre-commit-config.yaml`、`.buildkite/**`、`tools/**` | | xdist、共享 worker、helper 拆分/启动回滚、下载 fixture、进程池 | `OMNI-CI-2b` | `tests/conftest.py`、`tests/helpers/{client,clean,runtime,stage_config}.py`、`tests/model_tests/**` | | 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 6cbaaec4c..064c4282f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fe4a2a74` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3d1ad0c0` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2dd1a37de..e12f269a7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fe4a2a74` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 3d1ad0c0` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b8bb38f71..e19f49de4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fe4a2a74` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3d1ad0c0` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f15b19abd..551549929 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fe4a2a74` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3d1ad0c0` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index fcb288a3c..d1d15f794 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fe4a2a74` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3d1ad0c0` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5f133090c..1a7846538 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fe4a2a74` 验证存在 +- 源码校验：以上路径均已在 `main @ 3d1ad0c0` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 476d0f0dc..58bb2eae5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fe4a2a74`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3d1ad0c0`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 6f16f954 \ - --to fe4a2a74 \ + --from fe4a2a74 \ + --to 3d1ad0c0 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index 51168dd78..f4d9a0d92 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6523", "PR #6728", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_speech_stream.py, vllm_omni/entrypoints/openai/speech_usage.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_serving_speech_stream.py, tests/entrypoints/openai_api/test_tts_adapter.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] +sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6523", "PR #6728", "PR #6861", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_speech_stream.py, vllm_omni/entrypoints/openai/speech_usage.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_serving_speech_stream.py, tests/entrypoints/openai_api/test_tts_adapter.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] confidence: high --- @@ -198,3 +198,7 @@ Qwen 家族入口见 [Qwen-Omni](../qwen-omni/_index.md)。 EOS/非 length 正常通过、length+recorded-cap 拒绝、非流式一次 retry 及显式 seed/max 不 retry； raw/SSE/WebSocket 均携带 terminal metrics，并覆盖 partial/no-partial error。目标回归证明控制流和 丢弃语义，不证明任意 prompt 必达 EOS、音频质量或最佳 cap 比例。^[PR #6728] + + Ready CI 的 dummy-weight Base 即使到 192 tokens 仍可无 EOS 并返回 500，因而不是该 guard 的 + Ready oracle；此事实不改变 runtime、YAML、EOS 或音质合同，也没有 real-weight merge pass log。 + ^[PR #6861] ^[issue #6855] From 8baa9399ac2d97ec06f0502b8eb518237b025a4c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:49:22 +0800 Subject: [PATCH 407/465] knowledge(vllm-omni): sync b81aeb7b (PR #6828) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- knowledge/repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md | 5 ++++- 10 files changed, 16 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6551c83be..3d1c29e4a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fe4a2a7477da686d12dc20c1ab70fc266e90eb35 + previous_audited_sha: 3d1ad0c0f44425ad799b5d606a42d9b8a52c359c audited_ref: main - audited_sha: 3d1ad0c0f44425ad799b5d606a42d9b8a52c359c + audited_sha: b81aeb7b86837f6fe8956f3aef83798ad26c5a26 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ced255bff..a776038d2 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3d1ad0c0f44425ad799b5d606a42d9b8a52c359c`. The canonical machine baseline +b81aeb7b86837f6fe8956f3aef83798ad26c5a26`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 064c4282f..0315054f4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3d1ad0c0` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b81aeb7b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e12f269a7..e9669f0eb 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3d1ad0c0` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ b81aeb7b` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e19f49de4..11ed78c7c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3d1ad0c0` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b81aeb7b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 551549929..8b93f58e9 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3d1ad0c0` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b81aeb7b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d1d15f794..0e464166d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3d1ad0c0` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b81aeb7b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1a7846538..6c5443b1b 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3d1ad0c0` 验证存在 +- 源码校验：以上路径均已在 `main @ b81aeb7b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 58bb2eae5..5c90648ee 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3d1ad0c0`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b81aeb7b`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from fe4a2a74 \ - --to 3d1ad0c0 \ + --from 3d1ad0c0 \ + --to b81aeb7b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 9753dff91..24a058f17 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757", "PR #6529", "PR #6772", vllm_omni/experimental/fullduplex/openai/protocol.py] +sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6828", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/openai/chat_fallback.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/vad.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757", "PR #6529", "PR #6772", vllm_omni/experimental/fullduplex/openai/protocol.py] confidence: high --- @@ -271,6 +271,9 @@ confidence: high `--async-chunk`。不得用 deploy 默认值或参数 ID 猜实际模式。audio→text+audio case 若要稳定 验证 TTS，必须显式请求 text/audio modalities、`use_tts_template=True`、 `enable_thinking=False` 并断言目标关键词。 +- 对 `test_image_to_text_audio_001` 的 content-consistency oracle，test-local `_CI_DEPLOY` 只把 Stage-1 + Talker `temperature` 设为 `0.0`；shipping deploy/runtime 保持不变。该设置不证明 deterministic audio、 + 所有 flake 已消除或生产 sampling 修复；已放弃的 production sampling-mask 工作不属于本合同。^[PR #6828] ^[issue #6815] - duplex fixture 保留 Talker `max_model_len=4096`，但不覆盖三个 stage 的 KV sizing，不另造 eager-only server；protocol smoke 与 response tests 使用同一 server config。response-required 多轮测试每轮 重放同一 active-speech window，避免较短的 mid-utterance slice 合法地产生 LISTEN。reference audio From 0efa9aee5a3ae34153c372546d8b09a18ff9c2d1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:52:31 +0800 Subject: [PATCH 408/465] knowledge(vllm-omni): sync 705e781b (PR #6821) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../models/minicpm-o-4-5/rules-duplex.md | 25 ++++++++----------- 10 files changed, 23 insertions(+), 26 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3d1c29e4a..818df9da7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3d1ad0c0f44425ad799b5d606a42d9b8a52c359c + previous_audited_sha: b81aeb7b86837f6fe8956f3aef83798ad26c5a26 audited_ref: main - audited_sha: b81aeb7b86837f6fe8956f3aef83798ad26c5a26 + audited_sha: 705e781b0efc52bea75930ecd584690eafcc204a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a776038d2..eb0493fb0 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b81aeb7b86837f6fe8956f3aef83798ad26c5a26`. The canonical machine baseline +705e781b0efc52bea75930ecd584690eafcc204a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0315054f4..e39a3f740 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b81aeb7b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 705e781b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e9669f0eb..465f76ed7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b81aeb7b` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 705e781b` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 11ed78c7c..792726029 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b81aeb7b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 705e781b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8b93f58e9..973d83666 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b81aeb7b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 705e781b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0e464166d..114617e7c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b81aeb7b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 705e781b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 6c5443b1b..4771d7458 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b81aeb7b` 验证存在 +- 源码校验：以上路径均已在 `main @ 705e781b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5c90648ee..fbbdc5500 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b81aeb7b`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 705e781b`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 3d1ad0c0 \ - --to b81aeb7b \ + --from b81aeb7b \ + --to 705e781b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md index 931dbe238..affc0c18b 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules-duplex.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", "PR #6626", "PR #6767", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/deploy/minicpmo_4_5_2gpu.yaml, vllm_omni/deploy/minicpmo_4_5_3gpu.yaml, vllm_omni/deploy/minicpmo_4_5_8x4090.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py, "PR #6529", vllm_omni/experimental/fullduplex/openai/protocol.py, vllm_omni/experimental/fullduplex/openai/realtime_state.py, tests/entrypoints/openai/test_duplex_protocol.py] +sources: ["PR #6318", "PR #6346", "PR #6404", "PR #6458", "PR #6619", "PR #6630", "PR #6678", "PR #6626", "PR #6767", "PR #6821", vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, vllm_omni/deploy/minicpmo_4_5_2gpu.yaml, vllm_omni/deploy/minicpmo_4_5_3gpu.yaml, vllm_omni/deploy/minicpmo_4_5_8x4090.yaml, examples/online_serving/minicpmo/realtime_duplex_demo.py, vllm_omni/experimental/fullduplex/client.py, vllm_omni/experimental/fullduplex/minicpmo45/adapter.py, vllm_omni/experimental/fullduplex/minicpmo45/session.py, vllm_omni/experimental/fullduplex/minicpmo45/stage0.py, vllm_omni/experimental/fullduplex/openai/realtime_input.py, vllm_omni/experimental/fullduplex/openai/runtime_adapter.py, vllm_omni/experimental/fullduplex/openai/runtime_bridge.py, vllm_omni/experimental/fullduplex/openai/serving.py, vllm_omni/experimental/fullduplex/openai/session_runner.py, vllm_omni/experimental/fullduplex/video_stacking.py, tests/config/test_config_factory.py, tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/entrypoints/openai_api/test_duplex_handler.py, tests/examples/test_minicpmo_realtime_duplex_simple_demo.py, "PR #6529", vllm_omni/experimental/fullduplex/openai/protocol.py, vllm_omni/experimental/fullduplex/openai/realtime_state.py, tests/entrypoints/openai/test_duplex_protocol.py] confidence: high --- @@ -96,16 +96,13 @@ confidence: high - 触发：native duplex response create/end、input commit、playback ACK、conversation item register/delete/truncate 或 auto-response terminal closure。 -- 强制：active response 绑定它回答的 input commit sequence；如果 response 在 user-speaking 阶段创建， - 则绑定紧随其后的真实 commit。response-scoped snapshot 以 response ID 保存 assistant text、audio - text marks 与该 commit sequence。ACK 的 item 必须精确为 `item_`、确实属于已知 - assistant response，且不能晚于后续 input commit；mismatch/not-found/too-late 分别返回 typed error。 - register/delete/truncate 只消费并清理 matching response snapshot、pending marks/commit metadata。 - auto-response safety close 即使已没有 active response，也必须完成对应 model turn。 -- 禁止：用全局 last-assistant buffer 让晚到 ACK/truncate 绑定到另一 response；把前一 response history - 提交到下一 input；用 nonmatching item 清理 snapshot；把未 ACK/delete 的 snapshot 宣称已有有界 - retention。 -- 验收：覆盖 response-before/after-input commit、ACK identity/existence/late ordering、两个 response - 交错的 register/delete/truncate、pending audio marks、no-active auto close 与下一 fenced append；断言 - typed error 无副作用且只释放目标 snapshot。当前实现的 snapshots 在 ACK/delete 前仍可能无界。 - ^[PR #6529] +- 强制：pre-commit 对精确 `item_` 的 ACK 预留当前 history position；active zero-audio ACK + 也创建不可见 placeholder。response 完成后的 ACK 在该 slot upsert/extend，未预留 stale ACK 仍返回 typed + too-late error。partial snapshot 保留到 response inactive 且 playback fully committed；delete 清理 + placeholder、snapshot、marks/truncation state。explicit truncate 与 ACK truncate 施加 monotonic minimum + cap，later ACK 不得恢复 text/audio。client final input commit 前 checkpoint ACK，commit 后再 ACK draining + response-local playback；范围仅 MiniCPM native-duplex `ack_only`。 +- 禁止：用 global last-assistant buffer 或在新 user item 后 append old assistant；让 later ACK 越过 hard cap； + 把此 ordering fix 外推为通用 realtime/reliability guarantee。 +- 验收：zero-audio reserve、partial ACK→commit→response-end→later ACK 原位扩展、unreserved stale reject、 + delete cleanup、两种 truncate cap 与 checkpoint-before-commit 顺序均须覆盖。^[PR #6529] ^[PR #6821] From 42cf7c567e0296cbbdd2a9eaf952312e3180e2db Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:54:59 +0800 Subject: [PATCH 409/465] knowledge(vllm-omni): sync eb11446b (PR #6834) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/distributed/rules.md | 16 ++++++++-------- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 818df9da7..48e46a29f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b81aeb7b86837f6fe8956f3aef83798ad26c5a26 + previous_audited_sha: 705e781b0efc52bea75930ecd584690eafcc204a audited_ref: main - audited_sha: 705e781b0efc52bea75930ecd584690eafcc204a + audited_sha: eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index eb0493fb0..8976ca383 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -705e781b0efc52bea75930ecd584690eafcc204a`. The canonical machine baseline +eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e39a3f740..8257d063a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 705e781b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ eb11446b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 465f76ed7..7a40f38ad 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 705e781b` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ eb11446b` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 792726029..c5f4c6886 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 705e781b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ eb11446b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index 309c03339..756cb4c57 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-05 type: rule tags: [vllm-omni, components, distributed] -sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626", "PR #6529"] +sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", "PR #6834", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626", "PR #6529"] confidence: high --- @@ -79,7 +79,7 @@ confidence: high ## DIST-1g — 异步 processor 任务必须在入队时冻结边界状态 - 触发：`OmniChunkTransferAdapter.save_async` 向 custom next-stage processor 入队，且 producer request 会在 worker 发送前继续被 streaming update 原地修改。 -- 强制：只为 custom processor shallow-copy request，并递归复制其可变容器输入（additional information、prompt/token lists）；tensor leaves 保持共享。每个 queued segment boundary 在入队时移除 sender dedup watermark，使下一个 segment 能从零建立自己的 watermark；后台发送旧 task 不得清除新 segment 的 watermark。 +- 强制：只为 custom processor shallow-copy request，并递归复制其可变容器输入（additional information、prompt/token lists）；tensor leaves 保持共享。每个 queued segment boundary 在入队时移除 `requests_num_chunks_sent` chunk-progress watermark，使下一个 segment 能从 chunk 0 建立自己的 watermark；这不重置 `_segment_generation` gate，后台发送旧 task 也不得清除新 segment 的 watermark。^[PR #6834] - 禁止：给无 hook 的直通路径增加 request snapshot/copy；让 queued task 读取后来 mutation 的 token、metadata 或 output history；在后台 terminal cleanup 无条件删除可能已属于下一 segment 的 watermark。 - 验收：old/new segment 在 background flush 前交错时，新 watermark 保持；连续入队后各 processor 观察各自 admission-time request snapshot；无 custom processor 时 task 仍引用原 request。^[PR #6021] @@ -109,15 +109,15 @@ confidence: high - 触发：async-chunk sender dedup、segment terminal task、background cleanup 或同 request ID 的 streaming segment replacement。 - 强制：`save_async` 将 request 的 segment generation 冻结进 queued task；小于当前 generation 的 - late save 必须拒绝。terminal task 在后台 sender 执行前先把 generation 推进到下一段并清空旧 - chunk watermark，非 terminal preemption 才用 confirmed-token watermark 去重。request 最终 cleanup - 同时回收 generation 和 chunk state。 + late save 必须拒绝。terminal 必须清除 chunk-progress counter；仅当 request + `._omni_segment_generation is not None` 才把 expected generation 推进 `+1`，否则保持 current + generation，使 counter-less follow-up 仍以 current/0 enqueue。chunk watermark 只负责同段 progress + dedup，generation gate 负责跨段 stale fencing；request 最终 cleanup 同时回收 generation 和 chunk state。 - 禁止：只按裸 request ID 判断 late task 有效；让旧 terminal boundary 因旧 chunk count 被误拒； background completion 无条件删除下一段 watermark；把未 ACK 的 serving response snapshot描述成 此 sender generation 已给出有界 retention。 -- 验收：覆盖 old terminal 排队后 new segment 先开始、late old save、boundary 在 chunk counter reset - 后仍入队，以及普通 dedup/preemption；断言 old payload 不复活且 new generation/watermark 保留。 - ^[PR #6529] +- 验收：覆盖 counter-bearing terminal `+1` fencing 与 counter-less terminal hold/current-0 follow-up、 + counter clear、late old save 和普通 dedup/preemption。该变更不修复已有 stuck ID 或 abort 行为。^[PR #6529] ^[PR #6834] ## DIST-2a — diffusion EP group 必须满足运行中 MoE backend 的 communicator 合同 diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 973d83666..f0a84ec7b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 705e781b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ eb11446b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 114617e7c..91c699383 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 705e781b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ eb11446b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4771d7458..1f43b5ede 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 705e781b` 验证存在 +- 源码校验：以上路径均已在 `main @ eb11446b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index fbbdc5500..d238c09fc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 705e781b`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ eb11446b`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from b81aeb7b \ - --to 705e781b \ + --from 705e781b \ + --to eb11446b \ --repo \ --mode report-only ``` From d3d11aa75941902553db7168f3b208b67614c68b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:57:27 +0800 Subject: [PATCH 410/465] knowledge(vllm-omni): sync 8e26dce1 (PR #6701) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/models/boogu-image/_index.md | 5 +++-- .../vllm-omni/models/boogu-image/rules.md | 21 +++++++++++++++++++ knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/boogu-image/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 48e46a29f..61c541886 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 705e781b0efc52bea75930ecd584690eafcc204a + previous_audited_sha: eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c audited_ref: main - audited_sha: eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c + audited_sha: 8e26dce1f407bd96f7f01b7ced1ea7e019eead85 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 8976ca383..f0b59ad3b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c`. The canonical machine baseline +8e26dce1f407bd96f7f01b7ced1ea7e019eead85`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8257d063a..4bec8b94d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ eb11446b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8e26dce1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7a40f38ad..19fe4d264 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ eb11446b` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 8e26dce1` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c5f4c6886..7a308616d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ eb11446b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8e26dce1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f0a84ec7b..808455b98 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ eb11446b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8e26dce1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 91c699383..f46120fcb 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ eb11446b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8e26dce1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1f43b5ede..50cbe329d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ eb11446b` 验证存在 +- 源码校验：以上路径均已在 `main @ 8e26dce1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/boogu-image/_index.md b/knowledge/repos/vllm-omni/models/boogu-image/_index.md index a48572d15..69f81adab 100644 --- a/knowledge/repos/vllm-omni/models/boogu-image/_index.md +++ b/knowledge/repos/vllm-omni/models/boogu-image/_index.md @@ -1,10 +1,10 @@ --- title: "Boogu Image" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/boogu_image/] +sources: ["PR #6701", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/diffusion/models/boogu_image/, vllm_omni/entrypoints/utils.py] confidence: high --- @@ -20,6 +20,7 @@ confidence: high 共 5 个文件： - `vllm_omni/diffusion/models/boogu_image/` +- revision/component loader、Edit-Turbo checkpoint boundary：[rules](rules.md) ## 依赖的共享代码模块 diff --git a/knowledge/repos/vllm-omni/models/boogu-image/rules.md b/knowledge/repos/vllm-omni/models/boogu-image/rules.md new file mode 100644 index 000000000..6a7776600 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/boogu-image/rules.md @@ -0,0 +1,21 @@ +--- +title: "Boogu-Image 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #6701", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py, vllm_omni/entrypoints/utils.py, tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py, tests/entrypoints/test_utils.py, tests/e2e/online_serving/test_boogu_image_edit.py] +confidence: high +--- + +# Boogu-Image 规则 + +## BOOGU-1a — global revision 必须填入每个未显式设 revision 的 stage + +- 触发：全局/stage checkpoint revision handoff、Boogu component loader 或 Edit-Turbo recipe 变更。 +- 强制：global revision 只补入未显式声明 revision 的 resolved stage；effective revision 必须一致传播到 + transformer source/prefetch、scheduler、mLLM、processor 和 VAE loaders。 +- 禁止：覆盖 stage explicit revision，或只让一个 component 使用 global revision 而其余悄然跟 default branch。 +- 验收：分别覆盖 global fill 与 stage override，并断言六个 consumer 的 effective revision 一致。Edit-Turbo + 文档/测试仅覆盖 `Boogu/Boogu-Image-0.1-Edit-Turbo` 的 `hotfix-1k-20260708`、4 steps、guidance 1.0； + 不证明 distinct Turbo class、DMD/renoise、quality 或一般 registry claim。^[PR #6701] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d238c09fc..b09d6089f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ eb11446b`（2026-08-31）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8e26dce1`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 705e781b \ - --to eb11446b \ + --from eb11446b \ + --to 8e26dce1 \ --repo \ --mode report-only ``` From b6bdbaba9cfe5a239a10b78f06cd195c950a9a4b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 05:59:56 +0800 Subject: [PATCH 411/465] knowledge(vllm-omni): sync 7992c221 (PR #6484) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-engine-lifecycle.md | 15 +++++++++------ knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 21 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 61c541886..4023db606 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c + previous_audited_sha: 8e26dce1f407bd96f7f01b7ced1ea7e019eead85 audited_ref: main - audited_sha: 8e26dce1f407bd96f7f01b7ced1ea7e019eead85 + audited_sha: 7992c2213ef1b1502b1ebeb8890e5a835e9c5d60 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f0b59ad3b..21dad1995 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8e26dce1f407bd96f7f01b7ced1ea7e019eead85`. The canonical machine baseline +7992c2213ef1b1502b1ebeb8890e5a835e9c5d60`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4bec8b94d..84f9e022a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8e26dce1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7992c221` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 19fe4d264..04e549c57 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8e26dce1` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 7992c221` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7a308616d..3e21eee08 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8e26dce1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7992c221` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 808455b98..aec30e007 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8e26dce1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7992c221` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f46120fcb..d288c7b3f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8e26dce1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 7992c221` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 50cbe329d..1f0a3faf3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8e26dce1` 验证存在 +- 源码校验：以上路径均已在 `main @ 7992c221` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md index 4249c43a9..3221baaa8 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-engine-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5221", "Issue #4855", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/openai/api_server.py, tests/engine/test_membership_controller.py, tests/engine/test_orchestrator_event_driven.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py, "PR #6189", tests/entrypoints/test_async_omni_diffusion_config.py, "PR #6367", vllm_omni/outputs/output_processor.py, tests/engine/test_async_omni_engine_abort_ack.py, tests/entrypoints/test_async_omni_pause_sleep_routing.py] +sources: ["PR #4834", "PR #4905", "PR #4912", "PR #5221", "Issue #4855", "PR #5277", "PR #5682", "PR #5713", "PR #5746", "PR #5843", "PR #5957", "PR #6008", "PR #6084", "PR #6138", "PR #6202", "PR #6484", vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/membership_controller.py, vllm_omni/engine/messages.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/stage_pool.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_engine_startup.py, vllm_omni/engine/stage_runtime.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/stage_diffusion_client.py, tests/engine/test_membership_controller.py, tests/engine/test_orchestrator_event_driven.py, tests/engine/test_async_omni_engine_stage_init.py, tests/engine/test_single_stage_mode.py, tests/entrypoints/test_omni_base_profiler.py, "PR #6121", "PR #6214", "vllm_omni/engine/stage_runtime.py", "PR #5676", "PR #6525", "Issue #6435", "PR #5491", "PR #6033", "PR #5272", "PR #6186", "PR #6241", "vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py", "PR #6346", "PR #6581", tests/entrypoints/test_omni_sleep_mode.py, "PR #4092", vllm_omni/worker/base.py, "PR #6564", tests/engine/test_orchestrator.py, tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py, "PR #6189", tests/entrypoints/test_async_omni_diffusion_config.py, "PR #6367", vllm_omni/outputs/output_processor.py, tests/engine/test_async_omni_engine_abort_ack.py, tests/entrypoints/test_async_omni_pause_sleep_routing.py] confidence: high --- @@ -290,9 +290,12 @@ confidence: high - 禁止：只解析或记录 `runtime.env` 却不应用到本地 launch；把 stage 环境永久污染到父进程；用 runtime env scope 替代 diffusion device scope；将该本地 scope 推广到 remote replica 或 headless worker launcher。 - 验收：CPU/mock 回归分别覆盖 LLM 和 diffusion launch 看到 stage 值、父环境已有值恢复、原先未设置的 key 在异常后被清理，以及 diffusion launch 同时看到 stage device 值；确认 LLM launch lock 仍覆盖进程创建。^[PR #6214] -### SERV-9a — client diffusion batch width 不得覆盖 scheduler admission capacity +### SERV-9a — `max_num_seqs` 是唯一 diffusion admission width -- 触发：diffusion stage 初始化或 replica launch 接收客户端 `batch_size`，或修改其 engine config、scheduler capacity 与 stage startup 路径。 -- 强制：`initialize_diffusion_stage` 和 `launch_diffusion_stage_replica` 只将 `batch_size` 转发给 `StageDiffusionClient`；`build_diffusion_config()` 解析出的 `od_config.max_num_seqs` 必须保持 CLI `--max-num-seqs` 或 stage YAML 的值。两条启动路径和 single-stage 路径都不得把 client `diffusion_batch_size` 映射为 scheduler capacity。 -- 禁止：在 shared init path 写入 `od_config.max_num_seqs = batch_size`，或用 `max(CLI, batch_size)` 合并两个独立旋钮；也不得从 `max_num_seqs > 1` 推断具体 pipeline 自动支持 request batching，兼容 admission 仍受 `DIFF-6a` 约束。 -- 验收：分别 mock `initialize_diffusion_stage` 和 `launch_diffusion_stage_replica`，断言 config 的 `max_num_seqs` 保留 CLI/YAML 的 `1`、`8` 或 `None`，而 client 仍收到 `batch_size`；回归 single-stage 初始化。Wan request-level batching 必须显式设置 `--max-num-seqs N` 或 YAML；Qwen-Image step-execution 的并发 scheduler capacity 不得被默认 `diffusion_batch_size=1` 降为 1。^[PR #5676] ^[PR #6525] ^[Issue #6435] +- 触发：diffusion stage startup、scheduler admission width 或 legacy `diffusion_batch_size` caller。 +- 强制：`max_num_seqs` 是 CLI/YAML 到所有 startup paths 的唯一 admission width；legacy public + `diffusion_batch_size` kwarg 必须抛 actionable `TypeError`，不得 alias、drop 或做 precedence merge。 +- 禁止：把 width 当作 pipeline fusion/request-batch support；serial pipeline 在 width>1 时必须 init fail， + 实际 request batching 仍由 `DIFF-6a` structural compatibility/isolation 证明。删除 YAML comment 不是行为文档。 +- 验收：覆盖 inline/multiproc/single-stage 保留 configured width、legacy kwarg loud failure、serial + pipeline width>1 init reject，及真正 request-batch pipeline 的 DIFF-6a admission control。^[PR #6484] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b09d6089f..442e64e0e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8e26dce1`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7992c221`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from eb11446b \ - --to 8e26dce1 \ + --from 8e26dce1 \ + --to 7992c221 \ --repo \ --mode report-only ``` From cb9e50f9627c77263e8a94a5c83d2177db9e45b9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:02:15 +0800 Subject: [PATCH 412/465] knowledge(vllm-omni): sync a65e89cb (PR #6829) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../configuration/rules-topology-profiles.md | 22 +++++++++---------- .../components/configuration/rules.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4023db606..d74d2bf64 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8e26dce1f407bd96f7f01b7ced1ea7e019eead85 + previous_audited_sha: 7992c2213ef1b1502b1ebeb8890e5a835e9c5d60 audited_ref: main - audited_sha: 7992c2213ef1b1502b1ebeb8890e5a835e9c5d60 + audited_sha: a65e89cbd47fe7c9f39dba558200c0bc16a758f3 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 21dad1995..9d1a8d0a1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -7992c2213ef1b1502b1ebeb8890e5a835e9c5d60`. The canonical machine baseline +a65e89cbd47fe7c9f39dba558200c0bc16a758f3`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 84f9e022a..5d69de2fb 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7992c221` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ a65e89cb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md index c8908a933..4b5299fd3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, components, config] -sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_runtime.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/deploy/, "PR #6186", "PR #6813", "PR #6291", tests/config/test_config_factory.py, tests/config/test_pipeline_registry.py] +sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_runtime.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/deploy/, "PR #6186", "PR #6813", "PR #6291", "PR #6829", vllm_omni/config/pipeline_registry.py, tests/config/test_config_factory.py, tests/config/test_omni_config.py, tests/config/test_pipeline_registry.py, tests/utils/test_tracking_parser.py] confidence: high --- @@ -84,15 +84,15 @@ confidence: high - 禁止：在 CLI、factory 或 stage startup 为模型名硬编码 alias/resolver；把 alias 广播到所有 stage；因 diffusion stage 只有一个 replica 就隐式改变所有多 stage pipeline 的进程隔离。 - 验收：structured/legacy 两路覆盖 alias-only、canonical-only、相同/冲突值、错误 stage 与 unset control；用 root/partition/task path 证明 resolver 到达 stage model/tokenizer consumer，并用 single-stage、multi-stage opt-in/control 与 multi-replica veto 证明 exact inline predicate。PR #6813 只新增 default one/two-stage 参数化 test，其他组合仍须补验收。^[PR #5885] ^[PR #6813] -## CONF-5j — PipelineConfig 终端 stage 必须显式声明且不依赖位置 +## CONF-5j — PipelineConfig 在构造时必须验证终端 topology -- 触发：修改 `PipelineConfig.validate()`、静态 `OMNI_PIPELINES` registry，或新增/调整 +- 触发：修改 `PipelineConfig.__post_init__`、`get_validation_errors()`、静态 `OMNI_PIPELINES` registry，或新增/调整 multi-stage pipeline 的 `final_output` 声明。 -- 强制：保留既有「没有 entry stage」校验，并要求至少一个 stage 显式 - `final_output=True`；terminal 可以处于任意 position。静态 `PipelineConfig` registry - 必须逐项通过该 topology guard。 -- 禁止：从最后一个 stage、stage 数、entry/input、`final_output_type` 推断 terminal；不得把 - 此静态 guard 外推为 cycle 检测、resolver 配置、运行时或 E2E 行为已经验证。 -- 验收：无 terminal 的配置返回错误；terminal-not-last 通过；有效 fixture 显式标记 - terminal；所有静态 `PipelineConfig` 通过。resolver 生成的配置须在 materialize 后另测。 - ^[PR #6291] +- 强制：`__post_init__` 必须调用 `get_validation_errors()`，有 error 时以 formatted `ValueError` + eager fail；保留 nonempty stages、unique IDs、valid nonself inputs、entry 与任意位置 terminal + `final_output=True`。静态 registry 在构造时通过 guard；resolver callable 仅在 concrete object + materialize 时受 guard。 +- 禁止：保留 public `validate()` API 或 registry posthoc sweep；从最后 stage、stage 数、entry/input、 + `final_output_type` 推断 terminal；不得声称 resolver 后有独立 revalidation、cycle/runtime/E2E proof。 +- 验收：无 stages/terminal、duplicate ID、invalid/self input eager error；terminal-not-last 与有效 fixture + pass；static configs pass，resolver concrete result materialization guarded。^[PR #6291] ^[PR #6829] diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index 693e172c7..ec45a0fe6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -21,7 +21,7 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | PR 描述在做什么 | 精确规则组 | 第一批 live 源码 | |---|---|---| | strict schema、unknown field、alias、flat→nested、structured/legacy/direct parity、typed projection | `strict-normalization`：`VOMNI-CFG-1a`–`1h` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,strip_parent_engine_args}` → `vllm_omni/config/omni_config.py::{_build_diffusion_config_projection,VllmOmniConfig.from_pipeline_config}` → `vllm_omni/engine/stage_init_utils.py::{build_engine_args_dict,build_engine_args_dict_from_omni_stage_config}` | -| deploy YAML、`base_config`、pipeline/stage overlay、pipeline-owned alias/hook、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a`, [CONF-5i](rules-topology-profiles.md#conf-5i-pipeline-专属-alias-与-hook-必须在通用配置层声明归一并消费), [CONF-5j](rules-topology-profiles.md#conf-5j-pipelineconfig-终端-stage-必须显式声明且不依赖位置) | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,normalize_pipeline_cli_overrides,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::StageConfigFactory.create_from_model` → `vllm_omni/engine/stage_init_utils.py::_resolve_model_path` | +| deploy YAML、`base_config`、pipeline/stage overlay、pipeline-owned alias/hook、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a`, [CONF-5i](rules-topology-profiles.md#conf-5i-pipeline-专属-alias-与-hook-必须在通用配置层声明归一并消费), [CONF-5j](rules-topology-profiles.md#conf-5j-pipelineconfig-在构造时必须验证终端-topology) | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,normalize_pipeline_cli_overrides,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::StageConfigFactory.create_from_model` → `vllm_omni/engine/stage_init_utils.py::_resolve_model_path` | | MiniCPM-o shipping profile、duplex/chat 共服、Talker sampling floor | model-specific [`MCPMO-4e`](../../models/minicpm-o-4-5/rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → `merge_pipeline_deploy`；duplex adapter runtime sampling override | | composable strategy、axis、routing、load balancing、`strategy-config` | `composable-strategy`：`CONF-4a` | `vllm_omni/config/composable_parallel/strategy_loader.py::{parse_strategy_specs,load_strategy_specs}` → `translator.py::translate_strategy_stack` → `apply.py::apply_strategy_specs` → `config_factory.py::{StageConfigFactory._apply_strategy_specs,StageConfigFactory._reconcile_strategy_with_cli}` | | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 04e549c57..bbb032588 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7992c221` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ a65e89cb` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 3e21eee08..d577678a3 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7992c221` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ a65e89cb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index aec30e007..c495b2c83 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7992c221` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ a65e89cb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d288c7b3f..84ec6570b 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7992c221` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ a65e89cb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1f0a3faf3..8438ae529 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7992c221` 验证存在 +- 源码校验：以上路径均已在 `main @ a65e89cb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 442e64e0e..d15a5b945 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7992c221`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ a65e89cb`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 8e26dce1 \ - --to 7992c221 \ + --from 7992c221 \ + --to a65e89cb \ --repo \ --mode report-only ``` From 08a412674b6f30c280399d60549e8b41a4c00347 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:05:52 +0800 Subject: [PATCH 413/465] knowledge(vllm-omni): sync 49c6fc15 (PR #6553) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 3 ++- .../components/serving/rules-speech-output.md | 22 +++++++++++++++++++ .../vllm-omni/components/serving/rules.md | 1 + knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/qwen3-tts/_index.md | 5 ++++- 12 files changed, 40 insertions(+), 13 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-speech-output.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d74d2bf64..6217add10 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7992c2213ef1b1502b1ebeb8890e5a835e9c5d60 + previous_audited_sha: a65e89cbd47fe7c9f39dba558200c0bc16a758f3 audited_ref: main - audited_sha: a65e89cbd47fe7c9f39dba558200c0bc16a758f3 + audited_sha: 49c6fc15408ef92ee08619da6bf7ed4f8f7f96dd # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9d1a8d0a1..3d1e2db11 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -a65e89cbd47fe7c9f39dba558200c0bc16a758f3`. The canonical machine baseline +49c6fc15408ef92ee08619da6bf7ed4f8f7f96dd`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5d69de2fb..1b84f9086 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ a65e89cb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 49c6fc15` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index bbb032588..020429c8b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ a65e89cb` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 49c6fc15` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d577678a3..fe149536c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ a65e89cb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 49c6fc15` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c495b2c83..03f632ca6 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ a65e89cb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 49c6fc15` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 84ec6570b..852686623 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ a65e89cb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 49c6fc15` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8438ae529..4a81abeac 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ a65e89cb` 验证存在 +- 源码校验：以上路径均已在 `main @ 49c6fc15` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -39,3 +39,4 @@ sources: [] | assembled FastAPI route 覆盖、method/path ownership 或 request-body forwarding | [app assembly 规则](rules-app-assembly.md) | | stage config→EngineArgs projection、explicit `devices`、TP/local-DP/PP、replica layout 或 worker 创建前的 layout guard | [stage 启动与设备布局规则](rules-stage-startup.md) | | 公开请求字段的校验、来源冲突、alias/extras 归一与 consumer view | [请求输入合同](rules-request-input.md) | +| speech 输出采样率 capability、resample、streaming header/flush | [speech 输出规则](rules-speech-output.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md b/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md new file mode 100644 index 000000000..78bed551b --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-speech-output.md @@ -0,0 +1,22 @@ +--- +title: "Speech 输出采样率规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #6553", docs/serving/speech_api.md, vllm_omni/entrypoints/openai/audio_utils_mixin.py, vllm_omni/entrypoints/openai/protocol/audio.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/tts_adapters/base.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, tests/entrypoints/openai_api/test_audio_format.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py] +confidence: high +--- + +# Speech 输出采样率规则 + +## SERV-9b — speech `sample_rate` 必须由 adapter capability 限定 + +- 触发：public speech request `sample_rate`、TTS adapter capability、audio encoding 或 streaming output。 +- 强制：protocol 只接受 positive int；是否支持由 adapter capability 决定。Qwen3-TTS 只接受 `{8000,24000}`， + omitted 使用 native 24k，其他 adapter 对 supplied value reject。Qwen non-streaming encoding 前 resample； + batch default 可被 item override。streaming 只支持 mono integer downsample，保持 persistent filter state、 + source-rate consistency 与 flush tail，target header/meta 只在 audio 后发出且 first audio 必须有 source sr。 +- 禁止：upsample、16k/stereo/general adapter support，或据 server-side resampling 声称降低模型生成 latency/cost。 +- 验收：Qwen native/8k、other adapter reject、batch override、streaming chunk continuity/tail/source mismatch/ + first-audio source-sr failure。CPU/DSP 与单一 expansion fixture 不证明跨硬件音质、TTFB 或带宽收益。^[PR #6553] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 965686a4b..7f8067f7c 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -33,6 +33,7 @@ confidence: high | caller sampling params、pipeline `sampling_constraints`、stage config runtime extraction | `request-contract`：`SERV-4o` | `OmniBase.resolve_sampling_params_list` → `_get_sampling_constraints_list` → `_apply_sampling_constraints` | | OpenAI integer bounds、msgpack overflow、request-local cleanup | `request-contract`：`SERV-4p` | protocol request models → orchestrator stage dispatch → `_fail_request_client_error` | | SSE/streaming speech、audio format、PCM/WAV、speed、首 chunk 前校验 | `streaming-format`：`SERV-1a`, `SERV-1b` | `vllm_omni/entrypoints/openai/protocol/audio.py::{OpenAICreateSpeechRequest.validate_streaming_constraints,StreamingSpeechSessionConfig.validate_streaming_constraints}` → `serving_speech.py::{OmniOpenAIServingSpeech._validate_speech_streaming_request,OmniOpenAIServingSpeech.create_speech}` | +| speech `sample_rate`、adapter capability、resample 与 stream header | [SERV-9b](rules-speech-output.md#serv-9b-speech-sample_rate-必须由-adapter-capability-限定) | protocol → TTS adapter capability → `serving_speech.py` / `audio_utils_mixin.py` | | video reference 解码、mixed media、frame conversion/mux、bounded memory | `media-ingress`：`SERV-1c`–`1e`, `1k` | `entrypoints/openai/video_api_utils.py` decode/coerce/encode helpers → video server callers | | `ref_audio`、x-vector/ICL、content identity、artifact cache/readiness | `artifact-readiness`：`SERV-3a`–`3c` | `serving_speech.py` reference resolve/decode/cache → adapter speaker cache → prefix salt | | Prometheus、waiting/running gauge、replica stats、throttle、collector lifecycle、pipeline request gauge 或 image/diffusion metric emission | [metrics-lifecycle rules](rules-metrics.md)：`SERV-2a`, `SERV-2b`, `SERV-2e`, `SERV-2f` | `vllm_omni/entrypoints/omni_base.py::{OmniBase._log_summary_and_cleanup,OmniBase._process_stage_metrics_message,_publish_request_gauges}` → `vllm_omni/metrics/prometheus.py::{OmniPrometheusMetrics.__init__,set_running,set_waiting}` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d15a5b945..75492741b 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ a65e89cb`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 49c6fc15`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 7992c221 \ - --to a65e89cb \ + --from a65e89cb \ + --to 49c6fc15 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md index 28aafcb08..9e8b25e79 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md @@ -4,12 +4,15 @@ created: 2026-07-20 updated: 2026-09-05 type: index tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py] +sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6553", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py] confidence: high --- # Qwen3-TTS +- Public `sample_rate` 只接受 adapter 声明的 24 kHz native 或 8 kHz server-resampled output； + streaming 状态/header/flush 边界见 [SERV-9b](../../components/serving/rules-speech-output.md#serv-9b-speech-sample_rate-必须由-adapter-capability-限定)。^[PR #6553] + ## 名称与范围 - 正式 owner：Qwen3-TTS serving、ref audio、artifact cache、async Code2Wav From 097aabb9bfd042b4774e631a4a15dd1d1dc72de1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:11:56 +0800 Subject: [PATCH 414/465] knowledge(vllm-omni): sync e2b83db6 (PR #6556) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 7 ++++++- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/minimax-h3/_index.md | 4 +++- .../models/minimax-h3/rules-deployment.md | 14 ++++++++++++-- .../repos/vllm-omni/models/minimax-h3/rules.md | 1 + 13 files changed, 34 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6217add10..8906325a5 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: a65e89cbd47fe7c9f39dba558200c0bc16a758f3 + previous_audited_sha: 49c6fc15408ef92ee08619da6bf7ed4f8f7f96dd audited_ref: main - audited_sha: 49c6fc15408ef92ee08619da6bf7ed4f8f7f96dd + audited_sha: e2b83db683673a0d54bb58db5f12d27a7f9f20e0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3d1e2db11..3359a90f1 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -49c6fc15408ef92ee08619da6bf7ed4f8f7f96dd`. The canonical machine baseline +e2b83db683673a0d54bb58db5f12d27a7f9f20e0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 24268c7bf..a52356650 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, ci] -sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464", "PR #6730", "PR #6745", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742", "PR #6650", pyproject.toml, tests/helpers/mark.py, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, "PR #6174", .buildkite/cuda/test-weekly.yml, tests/e2e/offline_inference/test_dots_tts_expansion.py] +sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overview.md, tests/diffusion/quantization/test_svdquant_config.py, tests/diffusion/quantization/test_svdquant_linear.py, tests/diffusion/quantization/test_svdquant_tp_loading.py, tests/diffusion/quantization/test_wan_autoround_mxfp4.py, tests/e2e/offline_inference/test_wan21_autoround_mxfp4.py, "PR #5544", "PR #6162", "PR #6170", "PR #6303", "PR #6390", "PR #6613", .buildkite/cuda/test-nightly.yml, tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6054", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/e2e/online_serving/test_hunyuan_video_15_expansion.py, tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json, tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json, "PR #6349", .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, tests/helpers/clean.py, "PR #6704", tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py, "PR #5464", "PR #6730", "PR #6745", "PR #6727", tests/diffusion/quantization/test_quantization_quality.py, "PR #5831", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, "PR #6742", "PR #6650", pyproject.toml, tests/helpers/mark.py, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, "PR #6174", .buildkite/cuda/test-weekly.yml, tests/e2e/offline_inference/test_dots_tts_expansion.py, "PR #6556", tests/e2e/online_serving/minimax_h3/] --- # vLLM-Omni CI @@ -62,6 +62,11 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie recorded here; therefore it does not establish an H100 pass, a result on other hardware, a broader memory bound, or a runtime/quantization implementation change. ^[PR #6742 / issue #6735] +## MiniMax-H3 H100_2 merge matrix + +- FL2VA/Ref2VA DP2、Turbo 与 FastH3 由四个 spawn-isolated pytest processes 组成；精确 topology、 + timeout 与仅限 media-container 的 assertions 见 [MMH3-3m](../models/minimax-h3/rules-deployment.md#mmh3-3m-h100_2-merge-lane-只覆盖精确-h3-ci-matrix)。^[PR #6556] + ## MiniCPM native-duplex server-VAD coverage - Ready/merge 的 MiniCPM duplex source dependencies 纳入 server-VAD runner；其 L4 full-model diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1b84f9086..72cc9cfcb 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 49c6fc15` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e2b83db6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 020429c8b..026746978 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 49c6fc15` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ e2b83db6` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index fe149536c..312866708 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 49c6fc15` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e2b83db6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 03f632ca6..a0ed866cb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 49c6fc15` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e2b83db6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 852686623..e534b089c 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 49c6fc15` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e2b83db6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4a81abeac..76563814d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 49c6fc15` 验证存在 +- 源码校验：以上路径均已在 `main @ e2b83db6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 75492741b..97f1224a4 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 49c6fc15`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e2b83db6`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from a65e89cb \ - --to 49c6fc15 \ + --from 49c6fc15 \ + --to e2b83db6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 528555ccd..1544d066f 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -4,7 +4,7 @@ created: 2026-08-05 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742", "PR #6666", "PR #6824", "PR #6714"] +sources: ["PR #5703", "PR #5706", "PR #5709", "PR #5720", "PR #5723", "PR #5737", "PR #5740", "PR #5752", "PR #5756", "PR #5764", "PR #5779", "PR #5785", "PR #5801", "PR #5824", "PR #5829", "PR #5837", "PR #5840", "PR #5850", "PR #5857", "PR #5881", "PR #5885", "PR #5891", "PR #5896", "PR #5914", "PR #5946", "PR #5972", "PR #5978", "PR #5991", "PR #5863", "PR #6476", "PR #6555", "PR #6556", .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-ready.yml, .buildkite/cuda/test-merge.yml, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/, docs/design/architecture_overview.md, docs/models/supported_models.md, docs/user_guide/quantization/fp8.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/, vllm_omni/model_executor/models/minimax_h3/, vllm_omni/model_executor/stage_input_processors/minimax_h3.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/platforms/npu/platform.py, vllm_omni/platforms/rocm/platform.py, vllm_omni/quantization/int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/, tests/diffusion/quantization/test_int8_config.py, tests/e2e/accuracy/minimax_h3/, tests/e2e/features/comfyui/test_comfyui_integration.py, tests/e2e/online_serving/minimax_h3/, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, vllm_omni/entrypoints/openai/video_api_utils.py, "PR #6213", "PR #6742", "PR #6666", "PR #6824", "PR #6714"] confidence: high --- @@ -98,6 +98,8 @@ confidence: high 隔离见 MMH3-2h；它不替换 H3 的自定义 solver。 - FastH3 是 exact FastVideo artifact 的 startup-only t2va fusion；它不使用 Turbo/native dynamic manager，strict artifact/request/offload gates 见 MMH3-2n。^[PR #6714] +- H100_2 merge lane 的 FL2VA/Ref2VA/Turbo/FastH3 四个隔离 process 只构成精确 L3 media + contract，不能升级为 runtime 或性能支持声明；见 MMH3-3m。^[PR #6556] - step execution 是 H3 的模型专有 stateful path：只在 resolved attention backend 能隔离多个 packed documents 时合并 request；否则仍逐 request 前向。request state、rank-0 preparation、Cache-DiT/DLO/multi-output 排除与有界验证证据见 MMH3-2k，不能由 generic runner guard 推断 multi-rank failure synchronization。^[PR #5810] - LightX2V Turbo v1.0 仅支持 legacy dynamic LoRA；精确 artifact mapping、packed QKV/FC1 binding、active-request sampling 合同与 DLO-resident A/B sidecar 边界见 MMH3-2j；转换后的 mixed-rank diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md index c4209cfb6..29b098403 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-deployment.md @@ -1,10 +1,10 @@ --- title: "MiniMax H3 部署与证据规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", "PR #6555", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py", "PR #6072", "PR #6526", tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/offloader/test_module_residency.py] +sources: ["PR #5723", "PR #5764", "PR #5836", "PR #5850", "PR #5857", "PR #5863", recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md, "PR #5891", "PR #5896", "PR #5946", "PR #5969", "PR #5972", "PR #6555", "PR #6556", docs/models/supported_models.md, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-4090.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md, recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/utils/fa.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/platforms/npu/platform.py, .buildkite/cuda/test-merge.yml, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/e2e/online_serving/minimax_h3/, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5910", "PR #6213", "PR #6345", "tests/diffusion/models/minimax_h3/test_minimax_h3_vae_tiling.py", "PR #6072", "PR #6526", tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/offloader/test_module_residency.py] confidence: high --- @@ -253,3 +253,13 @@ measurement；模型输入、执行与加载合同返回 [MiniMax H3 rules](rule - 强制：`_encode_visual_conditions` 必须在同一个 `video_vae` residency window 内完成该请求的全部 image 与 video encode；`_encode_reference_audio_conditions` 必须在同一个 `audio_vae` residency window 内完成 embedded 与 standalone audio encode。resident helper 只能在对应 scope 内调用，scope 退出时必须释放 component，异常也必须清理。 - 禁止：按 image、video 或 audio 条目分别建立重复的 H2D/D2H lifecycle；在前一个 scope 结束后直接调用 CPU-bound VAE；把单次 image 的行为外推为混合 reference，或把 transfer-level A/B 结果写成完整模型峰值显存与通用性能保证。 - 验收：contract tests 覆盖单/多 image、image+video、embedded+standalone audio 以及 encode exception，精确断言每个 VAE 只有一次 activate/offload；另验证 H3 model-level offload 注册所有实际 DiT 与 VAE stage，并以固定硬件和 workload 单独复核真实性能与输出质量。^[PR #6072] + +## MMH3-3m — H100_2 merge lane 只覆盖精确 H3 CI matrix + +- 触发：H3 CUDA merge CI、DLO/DP/TP/USP/HSDP/TE/VAE-PP e2e fixture。 +- 强制：一个 `h100_2` 150m job 以四个独立 35m pytest spawn processes 运行：FL2VA TP1/DP2 两并发、 + Ref2VA image-only 同配置、Turbo FL2VA TP2+DLO/no-AllGather（五 sigma/四 eval）、FastH3 1024×576 + 四步 HSDP2/USP2/TE TP2/VAE PP2。断言仅同步 MP4 geometry/FPS/audio positive sample。 +- 禁止：把 CI collection 写成 runtime support/performance/general topology proof；final target 的 runtime + changes已 reverted，不得以它宣称 generic support。 +- 验收：核对四 process timeout/request 600s/spawn isolation 与上述 exact cases。^[PR #6556] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index b74a793b8..5e88dbce3 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -47,6 +47,7 @@ confidence: high | DGX Spark、GB10、unified memory、FP8、offload/OOM | [MMH3-3e](rules-deployment.md#mmh3-3e-gb10-unified-memory-容量证据不等于离散-gpu-offload-合同) | single-partition recipe → allocator/header evidence → output probe | | RTX PRO 6000、TP2、Ulysses、2/4/8 GPU、PCIe | [MMH3-3f](rules-deployment.md#mmh3-3f-rtx-pro-6000-scaling-只绑定单机-t2va-协议) | topology → exact warmed T2VA measurements → memory method | | Ascend packed varlen、LaserAttention、mask churn、E2E/HBM | [MMH3-3g](rules-deployment.md#mmh3-3g-ascend-mask-free-数字只绑定报告的-h3-packed-workload) | H3 opt-in → exact NPU topology/workload → kernel/E2E/memory evidence | +| H100_2 merge、FL2VA/Ref2VA DP2、Turbo、FastH3 | [MMH3-3m](rules-deployment.md#mmh3-3m-h100_2-merge-lane-只覆盖精确-h3-ci-matrix) | one CI job → four isolated pytest processes → bounded media assertions | ## MMH3-1b — joint video/audio quality 与 offload 兼容性必须一起验收 From ad2e9d5795eb9ad8c37637d1b66999ea944e4ba0 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:13:59 +0800 Subject: [PATCH 415/465] knowledge(vllm-omni): sync d1c8beeb (PR #6858) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/docs/_index.md | 15 +++++++-------- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 19 insertions(+), 20 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8906325a5..91c378103 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 49c6fc15408ef92ee08619da6bf7ed4f8f7f96dd + previous_audited_sha: e2b83db683673a0d54bb58db5f12d27a7f9f20e0 audited_ref: main - audited_sha: e2b83db683673a0d54bb58db5f12d27a7f9f20e0 + audited_sha: d1c8beeb180b4617437c460ba5c8f61a4a561a7e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3359a90f1..2a6b81f19 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e2b83db683673a0d54bb58db5f12d27a7f9f20e0`. The canonical machine baseline +d1c8beeb180b4617437c460ba5c8f61a4a561a7e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 72cc9cfcb..8a5265ad3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e2b83db6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d1c8beeb` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 026746978..56893f392 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e2b83db6` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ d1c8beeb` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 312866708..27c3d9133 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e2b83db6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d1c8beeb` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a0ed866cb..52b0f794b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e2b83db6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d1c8beeb` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e534b089c..aef35ba75 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e2b83db6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d1c8beeb` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 76563814d..174fa6e79 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e2b83db6` 验证存在 +- 源码校验：以上路径均已在 `main @ d1c8beeb` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/_index.md b/knowledge/repos/vllm-omni/docs/_index.md index 8d7708f2b..4c2c6363a 100644 --- a/knowledge/repos/vllm-omni/docs/_index.md +++ b/knowledge/repos/vllm-omni/docs/_index.md @@ -1,10 +1,10 @@ --- title: "vLLM-Omni 文档" created: 2026-07-10 -updated: 2026-09-02 +updated: 2026-09-05 type: index tags: [vllm-omni, docs] -sources: ["PR #5715", "PR #6029", .claude/skills/readme.md, README.md, docs/configuration/README.md, docs/contributing/README.md, docs/getting_started/installation/README.md, docs/getting_started/installation/gpu/cuda.inc.md, docs/getting_started/installation/npu/npu.inc.md] +sources: ["PR #5715", "PR #6029", "PR #6858", .claude/skills/readme.md, README.md, docs/configuration/README.md, docs/contributing/README.md, docs/getting_started/installation/README.md, docs/getting_started/installation/gpu/cuda.inc.md, docs/getting_started/installation/npu/npu.inc.md] --- # vLLM-Omni 文档 @@ -28,11 +28,10 @@ sources: ["PR #5715", "PR #6029", .claude/skills/readme.md, README.md, docs/conf ## 发布与安装边界 -- stable release 从 0.14 起跟随 upstream vLLM 的偶数 minor cadence；版本兼容按 matching - major/minor 判断。`vLLM-Omni 0.26.x` 要求 `vLLM 0.26.x`，稳定 0.26.0 文档固定 - vLLM `v0.26.0`，configuration 链接也必须指向 0.26 文档，不能拿旧版本示例混用。 -- CUDA 文档的 0.26 base image 是 `vllm/vllm-openai:v0.26.0`；NPU source install 另有 - 平台专属 tuple：vLLM `v0.26.0` + vLLM-Ascend `releases/v0.26.0rc`。Review 明确没有把 - NPU 更新扩散到 MUSA/quickstart，因此各平台 pin 必须回到自己的安装页核对。 +- stable 0.28 文档将 vLLM-Omni `0.28.x` 与 vLLM `0.28.x` 对齐，configuration/env links 指向 + v0.28；这只是 release-documentation guidance，不能从 README highlights 推出 runtime/model support。 +- NPU A2/A3 installation examples 使用 aligned vLLM-Ascend v0.28.0 images；latest-main 安装仍在该 + aligned container 内进行。CUDA nightly `nightly` 是 rolling tag，`nightly-` 为 reproducible + commit tag，文档称只保留 newest 14。各 tag/image 的 live availability 仍须发布时核验。^[PR #6858] - README 将 full-duplex realtime serving 标为 experimental；公开说明、验收和支持承诺 不能把它写成 stable feature。 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 97f1224a4..57c1419bf 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e2b83db6`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d1c8beeb`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 49c6fc15 \ - --to e2b83db6 \ + --from e2b83db6 \ + --to d1c8beeb \ --repo \ --mode report-only ``` From 23ec8c01e0e92fa75dca50359e618306adaf1419 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:18:09 +0800 Subject: [PATCH 416/465] knowledge(vllm-omni): sync 2298359f (PR #6645) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 3 +- .../rules-diffusion-attention-precedence.md | 37 +++++++++++++++++++ .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 10 files changed, 50 insertions(+), 12 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/configuration/rules-diffusion-attention-precedence.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 91c378103..bb2356293 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e2b83db683673a0d54bb58db5f12d27a7f9f20e0 + previous_audited_sha: d1c8beeb180b4617437c460ba5c8f61a4a561a7e audited_ref: main - audited_sha: d1c8beeb180b4617437c460ba5c8f61a4a561a7e + audited_sha: 2298359f677efdacc67c2924ed71df645529eab4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 2a6b81f19..d74bd6619 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d1c8beeb180b4617437c460ba5c8f61a4a561a7e`. The canonical machine baseline +2298359f677efdacc67c2924ed71df645529eab4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8a5265ad3..436625d18 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d1c8beeb` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2298359f` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 @@ -39,6 +39,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] | 对象存储 URI、HF cache snapshot 或空 `config.json` 的模型名称回退与 pipeline 路由 | [模型引用路由规则](rules-model-reference-routing.md) | | stage full-payload transport capability、topology-owned projection 或 deploy/CLI override rejection | [stage transport capability](rules-stage-transport.md) | | diffusion Ulysses SymmMem transport flag 的 deploy/CLI/default-stage projection | [diffusion parallel transport rules](rules-diffusion-parallel-transport.md) | +| diffusion attention shorthand、structured config 与 global override 的优先级和最终规范表示 | [diffusion attention precedence](rules-diffusion-attention-precedence.md) | | 审计配置来源和多层加工 | [config audit](config-audit-plain-language.md) | | 设计配置归一化与 parity 验证 | [config normalization parity](config-normalization-parity.md) | | 查询 `Omni()` 初始化参数 | [Omni init args](omni-init-args.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-diffusion-attention-precedence.md b/knowledge/repos/vllm-omni/components/configuration/rules-diffusion-attention-precedence.md new file mode 100644 index 000000000..73f1de257 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/configuration/rules-diffusion-attention-precedence.md @@ -0,0 +1,37 @@ +--- +title: "diffusion attention 配置优先级与规范表示" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, config, diffusion] +sources: ["PR #6645", "Issue #6644", vllm_omni/config/stage_config.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/stage_init_utils.py, tests/config/test_config_factory.py, tests/engine/test_async_omni_engine_stage_init.py, tests/engine/test_stage_engine_args.py] +confidence: high +--- + +# diffusion attention 配置优先级与规范表示 + +## Direct 代码快速入口 + +- **VOMNI-CFG-2d — shorthand 与 structured attention config 的 reconcile。** 先看 `stage_config.py::reconcile_diffusion_attention_overrides`，再沿 `StageConfig` 的 legacy/typed projections 到 `stage_init_utils.py::_finalize_engine_args_dict` 和 `OmniDiffusionConfig.from_kwargs`；全局注入的 guard 在 `async_omni_engine.py`。 + +## VOMNI-CFG-2d — diffusion attention shorthand 与 structured config 必须先 reconcile、后 canonicalize + +- 触发：diffusion stage 同时可从 deploy YAML、stage CLI 或 runtime/global engine args 接收 + `diffusion_attention_backend` shorthand、`diffusion_attention_config` structured config，或 + `fastvideo_vsa_topk` legacy knob 时。 +- 强制：把 shorthand 视为 structured config 的 `default`；同一个 shared + `reconcile_diffusion_attention_overrides()` 必须先用于 typed 与 legacy projection。CLI shorthand + 覆盖 YAML `default`，但保留 YAML `per_role`；只剩 default 时移除 structured 值。CLI structured + 则移除 YAML shorthand。仅含 `per_role` 的 YAML 可与 CLI shorthand 一起进入最终 fold。stage 的 + structured config **或** shorthand 只要 `getattr(..., None)` 为非 `None`，就拥有该 stage，阻止 + global attention injection。`_finalize_engine_args_dict()` 必须 pop 两个 legacy fields + `diffusion_attention_backend`、`fastvideo_vsa_topk`，并以它们和 config 调用 + `parse_attention_config(...)`，使 `OmniDiffusionConfig` 只收到一个 structured representation。 +- 禁止：用 `hasattr` 把声明但值为 `None` 的 field 当作 stage override；让 shorthand 与 + `config.default` 同时流到 parser；用 CLI shorthand 删除 `per_role`；让 typed、legacy 或 headless + projection 各自做不同 reconcile；据此推断任一 attention kernel/backend 已被支持或有性能收益。 +- 验收：覆盖 stage shorthand、CLI shorthand 覆盖 YAML default 且保留 `per_role`、per-role-only YAML + 与 shorthand 合并、CLI structured 清除 YAML shorthand，以及 stage 任一种非空表示都不接收 global + injection；对两种 projection 的输出经 `OmniDiffusionConfig.from_kwargs`/normalization 断言无 + mutually-exclusive conflict，且最终无 shorthand。现有证据是 config/engine unit tests 与 reviewer + 指出的 #6644 conflict path；不构成 kernel、硬件、端到端生成或 backend-support 证明。^[PR #6645] ^[Issue #6644] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 56893f392..725f1c532 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d1c8beeb` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 2298359f` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 27c3d9133..bae84f942 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d1c8beeb` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2298359f` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 52b0f794b..9b9db5e01 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d1c8beeb` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2298359f` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index aef35ba75..3313ef6b0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d1c8beeb` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2298359f` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 174fa6e79..8c549a596 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d1c8beeb` 验证存在 +- 源码校验：以上路径均已在 `main @ 2298359f` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 57c1419bf..b5710aec4 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d1c8beeb`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2298359f`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from e2b83db6 \ - --to d1c8beeb \ + --from d1c8beeb \ + --to 2298359f \ --repo \ --mode report-only ``` From 1e0f363a90ee52982ae4a004b61393c72b048121 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:20:53 +0800 Subject: [PATCH 417/465] knowledge(vllm-omni): sync 44c8d1c4 (PR #6114) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 4 +-- .../components/serving/architecture.md | 36 +++++++++++++++++-- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 10 files changed, 47 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bb2356293..1c80df597 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d1c8beeb180b4617437c460ba5c8f61a4a561a7e + previous_audited_sha: 2298359f677efdacc67c2924ed71df645529eab4 audited_ref: main - audited_sha: 2298359f677efdacc67c2924ed71df645529eab4 + audited_sha: 44c8d1c4491bb0640d5e6073f00b86b7ba1809ed # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d74bd6619..4b5bd9ac8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2298359f677efdacc67c2924ed71df645529eab4`. The canonical machine baseline +44c8d1c4491bb0640d5e6073f00b86b7ba1809ed`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 436625d18..1a7ecc189 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2298359f` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 44c8d1c4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 725f1c532..cf773bd5b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2298359f` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 44c8d1c4` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index bae84f942..d8faa6cc9 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2298359f` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 44c8d1c4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9b9db5e01..a381d6c97 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2298359f` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 44c8d1c4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 3313ef6b0..409ab16a0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2298359f` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 44c8d1c4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8c549a596..5ba929b3a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2298359f` 验证存在 +- 源码校验：以上路径均已在 `main @ 44c8d1c4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -27,7 +27,7 @@ sources: [] | 遇到什么 | 查看哪里 | |---|---| -| 理解入口到 engine 的边界 | [architecture](architecture.md) | +| 理解入口到 engine 的边界，选择 unified HTTP/WS endpoint、区分 wire protocol，或确认 PersonaPlex/JoyVL 是否是独立 server | [architecture 与 API route owner](architecture.md) | | 根据 PR 描述直达 request contract、stage sampling constraints、streaming format、artifact readiness、factory lifecycle 或 metrics 的规则组与第一批源码 | [rules 与代码地图](rules.md) | | Prometheus family、pipeline waiting/running gauge、stage/replica snapshot 或 collector lifecycle | [metrics 生命周期规则](rules-metrics.md) | | upstream launcher shutdown、renderer warmup 与多 replica compile-cache 隔离 | [upstream 兼容规则](rules-upstream-compat.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/architecture.md b/knowledge/repos/vllm-omni/components/serving/architecture.md index c942a64b0..25bf76eb9 100644 --- a/knowledge/repos/vllm-omni/components/serving/architecture.md +++ b/knowledge/repos/vllm-omni/components/serving/architecture.md @@ -1,10 +1,10 @@ --- title: "Serving 共享架构" created: 2026-07-10 -updated: 2026-09-02 +updated: 2026-09-05 type: architecture tags: [vllm-omni, components, serving] -sources: ["PR #5085", vllm_omni/entrypoints/, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, tests/entrypoints/openai_api/test_video_server.py] +sources: ["PR #5085", "PR #6114", vllm_omni/entrypoints/, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/video_api_utils.py, vllm_omni/engine/orchestrator.py, docs/serving/README.md, docs/serving/full_duplex_api.md, docs/serving/openpi_api.md, docs/serving/realtime_api.md, docs/serving/standalone_servers.md, docs/serving/streaming_video_output_api.md, tests/entrypoints/openai_api/test_video_server.py] --- # Serving 共享架构 @@ -38,6 +38,38 @@ PIL frames。当前 loader 调用固定 `backend="pyav"`；注册表复用不等 - 请求协议对象：声明字段与兼容扩展的第一层 owner。 - engine、pipeline、prompt 和 sampling 参数：Serving 转换结果的最终 consumer。 +## API server 路由与协议 owner + +`vllm serve --omni` 的 unified server 是 task route 的装配点，不是模型 capability +registry：每个 route 是否能被当前 model/deploy 使用仍由其 pipeline/adapter 决定。HTTP core 的 +task route 是 `POST /v1/chat/completions`、`/v1/audio/speech`、`/v1/audio/generate`、 +`/v1/images/generations`、`/v1/images/edits`、`/v1/videos`，以及仅供简单脚本/测试的 blocking +`POST /v1/videos/sync`；health/model discovery 是 `GET /health`、`GET /v1/models`。同一 server +还 owner chat/speech batch、voice CRUD、video job list/get/delete/content 与 +`POST /v1/omni/sleep|wakeup`。`POST /v1/completions` 是 inherited vLLM route，能否用于某 task +取决于模型，不是 Omni task endpoint 的替代承诺。 + +六条 WebSocket path 不能混用 wire protocol： + +| path | owner/协议边界 | +|---|---| +| `WS /v1/audio/speech/stream` | incremental text → speech audio | +| `WS /v1/video/chat/stream` | video frames → text/audio | +| `WS /v1/realtime` | turn-based JSON events，audio 是 base64 PCM | +| `WS /v1/duplex` | explicit duplex endpoint；缺少 eligible handler 时 unsupported | +| `WS /v1/realtime/video` | JSON control 加 binary fragmented MP4 (`m4s`) | +| `WS /v1/realtime/robot/openpi` | MessagePack with NumPy extension，不是 OpenAI Realtime JSON | + +`/v1/realtime?duplex=1` 是第三条 path 的 mode，不是第七条；只有 deployment/model 提供 eligible +duplex handler 才切换，缺少 handler 则落回 ordinary realtime。客户端应读取 `session.created` +capabilities，不能根据 URL 假定 duplex。PersonaPlex（`/`、`/health`、`/api/chat`、 +`/v1/audio/duplex`）和 JoyVL（`/health`、`/v1/models`、`/v1/chat/completions`、`/reset`、 +`/v1/streaming/reset`、`/v1/streaming/persona`）是各自独立 process；后者还是调用另一 +OpenAI-compatible backend 的 session orchestrator，二者都不属于 unified server route set。 + +这份 map 的证据是 PR #6114 的 docs、route registration/static review 与 focused protocol tests；它 +不证明所有 model 的 E2E、硬件行为、吞吐，或这些实验协议的长期稳定性。^[PR #6114] + ## 请求参数怎样流动 ```text diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b5710aec4..636296202 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2298359f`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 44c8d1c4`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from d1c8beeb \ - --to 2298359f \ + --from 2298359f \ + --to 44c8d1c4 \ --repo \ --mode report-only ``` From 40ff1a6ab94f1a7826db5d403d2caba1503e5c26 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:23:31 +0800 Subject: [PATCH 418/465] knowledge(vllm-omni): sync e1b2f589 (PR #6913) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/cosmos3/rules.md | 17 +++++++++++++++-- 10 files changed, 27 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 1c80df597..84da9300a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2298359f677efdacc67c2924ed71df645529eab4 + previous_audited_sha: 44c8d1c4491bb0640d5e6073f00b86b7ba1809ed audited_ref: main - audited_sha: 44c8d1c4491bb0640d5e6073f00b86b7ba1809ed + audited_sha: e1b2f589c12c015ba668bc2a1b31d33ab68d0bf9 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4b5bd9ac8..08b2c929a 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -44c8d1c4491bb0640d5e6073f00b86b7ba1809ed`. The canonical machine baseline +e1b2f589c12c015ba668bc2a1b31d33ab68d0bf9`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1a7ecc189..4cd2b8df9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 44c8d1c4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e1b2f589` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index cf773bd5b..ae7beed22 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 44c8d1c4` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ e1b2f589` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index d8faa6cc9..461c8e47a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 44c8d1c4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e1b2f589` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a381d6c97..ec6a4e738 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 44c8d1c4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e1b2f589` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 409ab16a0..a04cf9fd3 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 44c8d1c4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e1b2f589` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 5ba929b3a..60898e3fd 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 44c8d1c4` 验证存在 +- 源码校验：以上路径均已在 `main @ e1b2f589` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 636296202..3fcd2af2a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 44c8d1c4`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e1b2f589`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 2298359f \ - --to 44c8d1c4 \ + --from 44c8d1c4 \ + --to e1b2f589 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index 4c34d1231..995b76fbf 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -1,10 +1,10 @@ --- title: "Cosmos3 规则" created: 2026-07-20 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325"] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325", "PR #6913"] confidence: high --- @@ -23,6 +23,7 @@ confidence: high | session manager、UND K/V、`freqs_gen`、CFG branch、request ID | COSMOS-2c | `state_cosmos3_adapter.py` → `pipeline_cosmos3.py` → `transformer_cosmos3.py` | | online/offload/HSDP/VAE parallel 支持声明 | COSMOS-3a | capability 文档/recipe → 对应公开入口与实现路径 | | ROCm、MI350X、AITER、latency/peak memory | COSMOS-3b | recipe 的 measurement commit/protocol → `platforms/rocm/platform.py` backend gate → 目标配置复测 | +| formatted final prompt、`prompt_suffix`、rank-zero log level | COSMOS-6a | `diffusion/models/cosmos3/pipeline_cosmos3.py::Cosmos3OmniDiffusersPipeline` | 若描述只写 Cosmos3，先从 registry 的 class key 进入 pipeline；只有命中共享 RNG、graph 或 offload 机制时再加 [Diffusion owner](../../components/diffusion/rules.md)。 @@ -124,3 +125,15 @@ confidence: high - 强制：Transfer 必须根据控制输入和选定 resolution 从 `VIDEO_RES_SIZE_INFO` 选择 canonical `(height, width, aspect_ratio)` bucket；原始请求尺寸先保存到 `COSMOS3_TRANSFER_REQUESTED_SIZE_KEY`，仅用于冲突告警，不能改变控制输入选出的几何。Transfer JSON prompt、以及 I2V/V2V 生成 canvas 的 aspect ratio 元数据必须使用 canonical ratio；若请求或 JSON prompt 的 ratio 与 canvas 冲突，所有 rank 都重写元数据，只有 rank 0 输出 warning。非 canonical canvas 必须映射到最近的受支持 ratio，而不是使用 gcd 形成新标签。 - 禁止：把 Transfer 的任意 `size=HxW` 当作最终几何或宣称其会被采用；保留控制输入已覆盖的 stale `aspect_ratio`；将元数据重写也放在 `_is_rank_zero()` 条件内导致 rank 间 token ids 不一致；用非 canonical 的精确 gcd ratio 触发无意义的冲突告警。 - 验收：覆盖 Transfer 请求尺寸与控制 bucket 不一致、请求/JSON ratio 冲突、rank 0 与非 rank 0 的 warning/rewrite 行为，以及 `1104x816`、`832x468` 等近似 canvas 映射到 `4,3`、`16,9` 的回归；JSON prompt 解析后必须逐 rank 得到相同 canonical `aspect_ratio`。^[PR #6325] + +## COSMOS-6a — final prompt 仅在 rank zero 以 DEBUG 留存 + +- 触发：修改 Cosmos3 JSON/metadata prompt formatting、可选 `prompt_suffix`，或该 pipeline 的 + final-prompt observability。 +- 强制：formatting 与可选 suffix 完成后，rank zero 使用同一最终 prompt 和 generation 路径，但只以 + `logger.debug("Final prompt: '%s'", prompt)` 记录；非 rank zero 不发出该 message。 +- 禁止：恢复 INFO emission；把此处调级称为全局 prompt redaction/privacy guarantee；忘记 DEBUG + 仍会以 plaintext 输出，或借此推断其他 trace、error 或 logger 已修改。 +- 验收：rank-zero test 用含 sentinel 与 suffix 的 prompt 断言最终字符串和 generation 输入不变，且 + DEBUG 才有该条记录；non-rank control 无此 emission。target 未新增 log-level test，证据仅为作者 + 报告的 CPU unit/manual check，不能外推为其它模型、日志配置或生产运行结论。^[PR #6913] From 3f12197ca453eaca181a77f55f437d9be8b7ec08 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:27:47 +0800 Subject: [PATCH 419/465] knowledge(vllm-omni): sync 477083a1 (PR #6884) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 1 + knowledge/repos/vllm-omni/ci/rules-amd.md | 37 +++++++++++++++++++ knowledge/repos/vllm-omni/ci/rules.md | 34 +---------------- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 12 files changed, 52 insertions(+), 44 deletions(-) create mode 100644 knowledge/repos/vllm-omni/ci/rules-amd.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 84da9300a..4b2419fcb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 44c8d1c4491bb0640d5e6073f00b86b7ba1809ed + previous_audited_sha: e1b2f589c12c015ba668bc2a1b31d33ab68d0bf9 audited_ref: main - audited_sha: e1b2f589c12c015ba668bc2a1b31d33ab68d0bf9 + audited_sha: 477083a103bef1e2c5fab174a61c268c43e9238d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 08b2c929a..90f6b7e4c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e1b2f589c12c015ba668bc2a1b31d33ab68d0bf9`. The canonical machine baseline +477083a103bef1e2c5fab174a61c268c43e9238d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index a52356650..179892e17 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -209,5 +209,6 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie | 遇到什么 | 查看哪里 | |---|---| | 审查硬件 lane（含 MiniMax-H3 DLO DP2 ready smoke 的证据边界）、回归 fence、CI 工具供应链或 xdist/shared fixture | [CI rules](rules.md) | +| AMD/ROCm timeout、quarantine、memory signal、Qwen3-TTS argv 或 Qwen3-Omni control-plane fixture | [AMD/ROCm CI rules](rules-amd.md) | | 查看仓库特有 CI 陷阱 | [CI guides](guides/_index.md) | | 调查历史 CI 失败 | [CI incidents](incidents/_index.md) | diff --git a/knowledge/repos/vllm-omni/ci/rules-amd.md b/knowledge/repos/vllm-omni/ci/rules-amd.md new file mode 100644 index 000000000..2db03510d --- /dev/null +++ b/knowledge/repos/vllm-omni/ci/rules-amd.md @@ -0,0 +1,37 @@ +--- +title: "AMD/ROCm CI 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, ci] +sources: ["PR #6704", "PR #6830", "PR #6884", .buildkite/amd/, tests/helpers/clean.py, tests/helpers/stage_config.py, tests/buildkite/test_amd_pipeline.py, tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py] +confidence: high +--- + +# AMD/ROCm CI 规则 + +## OMNI-CI-2f — AMD CI stabilization 必须保留有界执行与测量信号 + +- 触发：AMD/ROCm Buildkite bootstrap、long-running shard timeout/quarantine、GPU-memory assertion、 + cleanup diagnostics、Qwen3-TTS argv，或 Qwen3-Omni sleep/abort control-plane E2E 变更。 +- 强制：debug-only READY+MERGE composition 保留 normal branch selection、一个 shared build dependency + 和 named groups；MI300 long steps 有 explicit bounds，已知 failing case 保持独立、time-bounded + `NonBlocking`。memory comparisons 在 model/offload 后采样、减每 run initial device use 并清 cache； + optional cleanup diagnostic 也必须 bounded。ROCm offload threshold 只是一项机制 signal，不是 + portability/capacity/performance claim。^[PR #6704] +- 强制：MI300 blocking layerwise-offload step 只 deselect exact Stable Audio parameter，并以 dependent + `mi300_1` `NonBlocking` step 保留该 case；其余 params、spawn/log env 不变。ROCm BF16 AITER + cross-attention 仅可用 `assert_close(rtol=1e-2, atol=1.5e-2)`，CUDA/XPU 保持 strict。^[PR #6830] +- 强制：AMD Qwen3-TTS Base step 是 direct `pytest`（无 nested `bash -c`），列出两个 Base test files， + marker 精确为 `advanced_model and cuda`、`--run-level` 为 `advanced_model`，保留 env/timeouts；这只证明 + rendered argv/collection，不证明 CUDA/ROCm runtime。Qwen3-Omni control-plane E2E 使用命名 + `ci/qwen3_omni_moe_colocate_async.yaml`：thinker-only stage 0、single GPU、8K、`max_num_seqs=1`、 + eager、no prefix cache。CUDA 保留 `.9` percentage/no byte cap；ROCm 仅该 MI300-ish fixture 清 percentage + 并 pin 2 GiB KV。test marks 是 `advanced_model`+`omni`、one-card `H100`/`MI325`。^[PR #6884] +- 禁止：把 debug override 当默认 routing、用邻近 green shard/NonBlocking/旧 head 宣称 target case + resolved，或让 diagnostic hang bypass timeout；不得把 absolute device use 当 model peak、ROCm tolerance + 扩展到其它平台/path/quality/runtime parity，或把 2 GiB 外推为生产/其他 model/SKU sizing。 +- 验收：render merge/ready/combined/malformed selection、timeouts/quarantine/dependency；分别检验 ROCm + sampling/cleanup 与 model assertion。PR #6884 必须 parse argv 与 platform overlay，断言 marker/run-level、 + CUDA preserve、ROCm-only clear/pin；最终需 exact final-head real MI300 L3 跑完两个 target jobs。该 PR + 未提供该 run，现有证据仅 author-reported local/static validation。^[PR #6704] ^[PR #6830] ^[PR #6884] diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 6ba65daf4..cf849e371 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-05 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818", "PR #6830", tests/diffusion/conftest.py, tests/diffusion/attention/test_flash_attn.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818", "PR #6830", "PR #6884", tests/diffusion/conftest.py, tests/diffusion/attention/test_flash_attn.py, tests/buildkite/test_amd_pipeline.py, tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py] confidence: high --- @@ -25,7 +25,7 @@ confidence: high | 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | | diffusion tiny builder、model settings、alignment exclusion、重模型 OOM | `OMNI-CI-2d` | `tests/model_tests/diffusion/{diff_model_builders,model_settings,test_alignment}.py` → common offline tests | | perf baseline、hardware label、DFX result artifact、assert-baseline | `OMNI-CI-3a` | `tests/dfx/conftest.py`、`tests/dfx/perf/scripts/run_benchmark.py`、`run_diffusion_benchmark.py`、`tests/dfx/perf/tests/**` | -| AMD/ROCm CI timeout、quarantine、diagnostic hang 或 memory assertion | `OMNI-CI-2f` | `.buildkite/amd/**` → target diffusion/TTS test → `tests/helpers/clean.py` | +| AMD/ROCm CI timeout、quarantine、diagnostic hang、memory assertion、Qwen3-TTS argv 或 Qwen3-Omni sleep/abort control plane | [OMNI-CI-2f](rules-amd.md#omni-ci-2f-amd-ci-stabilization-必须保留有界执行与测量信号) | `.buildkite/amd/**` → `tests/buildkite/test_amd_pipeline.py` / `ci/qwen3_omni_moe_colocate_async.yaml` → target test | ## OMNI-CI-1a — 硬件 lane 必须真实收集并执行目标路径 @@ -201,36 +201,6 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload - 验收：image tag 与目标 release 一致；startup 通过共享内存 preflight；目标 lane 证明实际 grader device；每个残余失败独立记录，尤其数值/质量回归不能由 API 兼容测试替代。^[PR #5976] -## OMNI-CI-2f — AMD CI stabilization must preserve bounded execution and the measurement signal - -- 触发：AMD/ROCm Buildkite bootstrap、long-running shard timeout/quarantine、GPU-memory assertion，或 - cleanup diagnostics 变更。 -- 强制：debug-only READY+MERGE suite composition must retain the normal branch selection and render one - shared build dependency with named groups; long-running MI300 steps need explicit Buildkite bounds. A known - failing case stays separately `NonBlocking` and time-bounded until its exact failure is resolved, rather than - remaining inside a blocking shard. -- 强制：memory comparisons start sampling after model/offload construction, subtract each run's initial device - usage, and clean retained cache state between runs. ROCm offload thresholds must remain conservative signals for - the asserted mechanism, not portability, capacity, or performance claims. Optional cleanup diagnostics must use - a bounded command that cannot turn a test failure into an unbounded hang. -- 禁止：把 READY/MERGE debug override 当成默认 lane routing；用邻近 shard 的绿色结果解除已知 case 的 - quarantine；把 absolute device use（含前一 topology 的 compiler/workspace cache）当模型峰值；或让 - diagnostic-only `amd-smi` hang bypass subprocess timeout and block pytest teardown. -- 验收：render `merge`、`ready`、combined and malformed/duplicate debug selections; prove MI300 long shards - have explicit timeouts and the unresolved TTS case is non-blocking. On ROCm, check the selected test's per-run - baseline/steady-state sampling and cleanup command separately from the model assertion; a local or different - machine threshold result is not a CI-wide performance verdict. ^[PR #6704] -- 强制：MI300 blocking layerwise-offload step 只 deselect exact - `stabilityai/stable-audio-open-1.0` parameter；同一 case 在独立 `mi300_1`、依赖 `amd-build` 的 - `NonBlocking` step 运行，其余 test parameters、spawn/log env 保持。ROCm BF16 AITER - FlashAttention-vs-SDPA cross-attention 可用 elementwise `assert_close(rtol=1e-2, atol=1.5e-2)`； - CUDA/XPU 仍保留各自 strict checks。 -- 禁止：用邻近 shard 或绿色 NonBlocking case 宣称 Stable Audio memory-savings issue 已解决；不得把 - ROCm tolerance 推广到其他平台、attention path、输出质量或 runtime parity。 -- 验收：render exact deselection node ID、quarantine command、dependency 与 `NonBlocking` wiring，且 - blocking test file 保留其他 parameters。ROCm test 保留 masked/unpadded cross-attention coverage，并 - 以超出 rtol/atol 的 mutation 证明仍会失败。^[PR #6830] - ## OMNI-CI-3a — DFX baseline artifact 与性能回归 gate 是两个合同 - 触发：修改 perf JSON 的 `baseline`、硬件 label/marker、benchmark result schema，或恢复性能阈值断言。 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 4cd2b8df9..a8f3b658c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e1b2f589` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 477083a1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ae7beed22..e700616c6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e1b2f589` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 477083a1` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 461c8e47a..9e117bd5a 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e1b2f589` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 477083a1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index ec6a4e738..af1187280 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e1b2f589` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 477083a1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a04cf9fd3..ce51219ed 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e1b2f589` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 477083a1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 60898e3fd..48f732cb7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e1b2f589` 验证存在 +- 源码校验：以上路径均已在 `main @ 477083a1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3fcd2af2a..ba9bb5d92 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e1b2f589`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 477083a1`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 44c8d1c4 \ - --to e1b2f589 \ + --from e1b2f589 \ + --to 477083a1 \ --repo \ --mode report-only ``` From 23193674f31296d9ae67b68c89851a7e5c9835db Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:31:49 +0800 Subject: [PATCH 420/465] knowledge(vllm-omni): sync 0288c3f5 (PR #6516) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../environment-variable-contract.md | 7 ++-- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-lora.md | 4 ++ .../diffusion/rules-system-runtime.md | 3 ++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- .../vllm-omni/models/sensenova-u1/_index.md | 9 +++-- .../vllm-omni/models/sensenova-u1/rules.md | 40 +++++++++++++++++++ 14 files changed, 68 insertions(+), 19 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/sensenova-u1/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4b2419fcb..3f363bfec 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e1b2f589c12c015ba668bc2a1b31d33ab68d0bf9 + previous_audited_sha: 477083a103bef1e2c5fab174a61c268c43e9238d audited_ref: main - audited_sha: 477083a103bef1e2c5fab174a61c268c43e9238d + audited_sha: 0288c3f56eb4a4ff410194e586498e3bc8ff8362 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 90f6b7e4c..a7d5aa46b 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -477083a103bef1e2c5fab174a61c268c43e9238d`. The canonical machine baseline +0288c3f56eb4a4ff410194e586498e3bc8ff8362`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a8f3b658c..bed255b3d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 477083a1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0288c3f5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md index b1de7109b..3dd9d020a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md +++ b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md @@ -1,10 +1,10 @@ --- title: "环境变量合同与诊断边界" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: guide tags: [vllm-omni, config, environment] -sources: ["PR #6217", "PR #6631", "PR #5221", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] +sources: ["PR #6217", "PR #6631", "PR #5221", "PR #6516", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] --- # 环境变量合同与诊断边界 @@ -21,6 +21,7 @@ sources: ["PR #6217", "PR #6631", "PR #5221", vllm_omni/config/environment_varia - `environment_variable_inventory.py` 是**分类清单，不是值解析器**：实际 consumer 继续拥有默认值、解析、优先级和读取时机。PR #5221 将 `VLLM_OMNI_EVENT_DRIVEN_ORCH` 加入 `PUBLIC_OMNI`（24 个名称）：只在 `Orchestrator` 构造时读取，默认 `0`；trim/lower 后仅 `1`、`true`、`yes`、`on` 开启，其他值保留 legacy loop。它同时选择 LLM orchestration 与 serving final-output drain，部署时必须在运行 orchestrator 的 stage-0 进程启动前设置；不是 request-level 开关，且不使 diffusion output 变为无轮询。新公共 Omni 名称必须以 `VLLM_OMNI_` 开头，旧前缀仅为兼容保留。^[PR #5221] - PR #6631 处理 #6217 合并后的 main 漂移：`VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` 是公开、实验性、仅环境变量的 float 秒数开关，默认 `600`；diffusion engine 在每次 `step_streaming` 请求路径读取它，非 float 或 `<=0` 时仅 warning-once 并回退默认值。它等待 async output，不是 `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` 的 full-payload input-coordinator 控制。`ROSVOT_SOURCE_DIR` 已随 SoulX-Singer 支持移除而从 model-specific `deprecate_remove` 清单删除。 +- PR #6516 将 `VLLM_OMNI_SENSENOVA_PAGED_DECODE` 加入 model-specific `promote` inventory（33→34），不是 `PUBLIC_OMNI`（仍为 24）。consumer 默认只 literal `"1"` 启用 model-local SenseNova single-sequence paged decode；这不使它成为 scheduler、prefix-cache 或通用 paged-KV setting。^[PR #6516] - 公开设置没有全局的 env-vs-CLI/YAML/request 优先级。按每个 consumer 的合同核实：请求可变行为应属于 request schema，稳定 model/stage 行为应属于 typed config，环境变量只适合启动、平台集成、诊断或兼容回退。import/startup 时读取的值必须在启动 CLI/导入前设置；启动后的父 shell 变化不会更新既有 worker。 - stage 的 `env`（以及兼容的 `runtime.env`）仅在启动该 stage/child process 时临时应用：键和值会字符串化，child 可继承，随后恢复父进程旧环境。因此它不能追溯改变已在此前 import 缓存的值，也不应承载 request-varying 语义。日志和诊断只输出键，绝不输出 stage `env` 的值。 - `collect_env.py` 只输出安全的公共 Omni allowlist、注册的 vLLM 名称和选择的平台前缀；显式 redact 的 `HF_TOKEN`、`HUGGINGFACE_HUB_TOKEN`、`OPENAI_API_KEY` 及常见 secret-like 名称不得输出值。为了诊断不完整/损坏的 Omni 安装，清单导入失败时 collector 回退为空清单而仍可报告既有安全信息。 @@ -30,7 +31,7 @@ sources: ["PR #6217", "PR #6631", "PR #5221", vllm_omni/config/environment_varia 1. 先沿真实 consumer 确认解析、默认、优先级与读取时机；不要把 inventory 或文档表当成运行时实现。 2. 新的稳定公开 Omni 设置同时更新分类、公开参考覆盖和 prefix gate；模型专用、benchmark、platform/external、internal 名称保持各自 owner，不因被读取而自动公开。 -3. 对 model-specific name 指定迁移目的（typed config、request scope、external、internalize 或 deprecate/remove）但不要把目的当成已经实现。目标提交的 56 个 model-specific 项为 promote 33、request scope 6、external 0、internalize 11、deprecate/remove 6；#6232 仍按模型族和 owner 跟踪迁移，尚不是现有运行时保证。该 issue 的旧 seven-item removal checklist 含已退休的 `ROSVOT_SOURCE_DIR`，不能覆盖当前 inventory。 +3. 对 model-specific name 指定迁移目的（typed config、request scope、external、internalize 或 deprecate/remove）但不要把目的当成已经实现。目标提交的 57 个 model-specific 项为 promote 34、request scope 6、external 0、internalize 11、deprecate/remove 6；#6232 仍按模型族和 owner 跟踪迁移，尚不是现有运行时保证。该 issue 的旧 seven-item removal checklist 含已退休的 `ROSVOT_SOURCE_DIR`，不能覆盖当前 inventory。 4. 变更 collector 或 stage `env` 时，以真实 worker/inheritance 与 redaction 测试验收；错误安装场景不得因可选清单导入而使诊断工具本身失效。 ## 已知证据限制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e700616c6..763ef02b7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 477083a1` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 0288c3f5` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md index 0be6c4a88..54d77f48c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -66,6 +66,10 @@ confidence: high alpha 与 unknown key；真实 smoke 另需固定 adapter、模型 revision、sampling 参数与输出比较。 当前 CPU evidence 使用 TP=1 `nn.Linear`；图片/视频附件是人工示例，没有 TP/HSDP/量化、 自动质量阈值或性能测量，不能作为持续 correctness gate。^[PR #2783] +- 模型例外：SenseNova-U1.5 distilled adapter 是 startup-only exactly-one-file one-way fusion；它以 + suffix/exact kohya mapping、fp32 delta 与 parameter `weight_loader` 做 TP sharding，zero-match fail， + 成功只留 sentinel，partial fuse 失败 abort/reload，不适用本规则的 unload/reversible contract。见 + [SENSENOVA-1a](../../models/sensenova-u1/rules.md#sensenova-1a-u15-distilled-lora-是启动期单文件单向融合)。^[PR #6516] ## DIFF-2n — Wan 多文件必须逐位置绑定 architecture-declared transformer diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index 7b39b26ec..dc13fd25d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -70,6 +70,9 @@ confidence: high 却继续 dense；直接换用 upstream backend 丢失 Omni 能力。 - 验收：unsupported platform/backend fail-fast；CUDA/NPU 对应 lane 和正式模型 E2E 证明 resolved backend；mixed-mask piecewise/SP 回归通过。 ^[PR #5543] ^[PR #6102] +- SenseNova-U1 paged AR decode 是 model-local, single-sequence CUDA fallback/capture，不是本规则的 + scheduler-owned paged backend；其 eligibility、growth、sleep 与 dynamic-PEFT veto 见 + [SENSENOVA-1b](../../models/sensenova-u1/rules.md#sensenova-1b-paged-ar-decode-是单序列模型本地-cuda-opt-in)。^[PR #6516] ## DIFF-4f — paged attention metadata 不从 padding 猜逻辑布局 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9e117bd5a..0ffa029dc 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 477083a1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0288c3f5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index af1187280..1ab035ad3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 477083a1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0288c3f5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ce51219ed..19bcace06 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 477083a1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0288c3f5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 48f732cb7..88243c6e3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 477083a1` 验证存在 +- 源码校验：以上路径均已在 `main @ 0288c3f5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ba9bb5d92..3ea5accd8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 477083a1`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0288c3f5`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from e1b2f589 \ - --to 477083a1 \ + --from 477083a1 \ + --to 0288c3f5 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/sensenova-u1/_index.md b/knowledge/repos/vllm-omni/models/sensenova-u1/_index.md index cc14532f8..a46afd75b 100644 --- a/knowledge/repos/vllm-omni/models/sensenova-u1/_index.md +++ b/knowledge/repos/vllm-omni/models/sensenova-u1/_index.md @@ -1,15 +1,15 @@ --- title: "SenseNova-U1（统一 LLM 即编码器即去噪器,无 VAE）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/diffusion/models/sensenova_u1/, vllm_omni/diffusion/registry.py] +sources: ["PR #6516", vllm_omni/diffusion/models/sensenova_u1/, vllm_omni/diffusion/registry.py] --- # SenseNova-U1 -以下事实在 `main @ 5d44868e` 复核（源码派生页,尚无本模型的运行经验沉淀）。 +U1.5 的 checkpoint/fusion 和 model-local decode 合同见 [rules](rules.md)。 ## 名称与范围 @@ -17,7 +17,7 @@ sources: [vllm_omni/diffusion/models/sensenova_u1/, vllm_omni/diffusion/registry （`sensenova_u1`, `pipeline_sensenova_u1`）,post `get_sensenova_u1_post_process_func`。单 stage diffusion,引擎默认 stage 配置（[Config 组件](../../components/configuration/architecture.md)）。无 deploy YAML。 -- 无变体有据,无 deploy YAML,树内未 pin checkpoint。 +- U1.5 `sensenova/SenseNova-U1.5-8B-MoT` 共享 pipeline；无 deploy YAML。它的 distilled adapter 是启动期单向融合，不是新增 pipeline/registry row。 - 源码：`pipeline_sensenova_u1.py`（66 KB,内含 `NEOVisionModel`/ `TimestepEmbedder`/`ConvDecoder`/`SenseNovaU1DenoisingAdapter`）+ `sensenova_u1_transformer.py`（32 KB）+ `fused_rmsnorm_rope.py` @@ -36,4 +36,5 @@ sources: [vllm_omni/diffusion/models/sensenova_u1/, vllm_omni/diffusion/registry ## 什么时候查这里 - 审查 sensenova_u1 的统一编码/去噪路径、融合核或权重映射改动。 +- 审查 U1.5 distilled LoRA、paged AR decode、GQA/RoPE/mask 时使用本页上方规则入口。 - 共享 RNG/graph 规则见 [Diffusion rules](../../components/diffusion/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/sensenova-u1/rules.md b/knowledge/repos/vllm-omni/models/sensenova-u1/rules.md new file mode 100644 index 000000000..0dc1ed0ad --- /dev/null +++ b/knowledge/repos/vllm-omni/models/sensenova-u1/rules.md @@ -0,0 +1,40 @@ +--- +title: "SenseNova-U1 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #6516", "Issue #6471", recipes/SenseNova/SenseNova-U1.5.md, vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py, vllm_omni/diffusion/models/sensenova_u1/paged_decode.py, vllm_omni/diffusion/models/sensenova_u1/sensenova_u1_transformer.py, tests/diffusion/models/sensenova_u1/] +confidence: high +--- + +# SenseNova-U1 规则 + +## Direct 代码快速入口 + +| 信号 | 规则 | 第一批源码 | +|---|---|---| +| U1.5 distilled 8-step LoRA、kohya keys、one-way fusion | SENSENOVA-1a | `pipeline_sensenova_u1.py::load_lora_weights` → loader → parameter `weight_loader` | +| `VLLM_OMNI_SENSENOVA_PAGED_DECODE`、think/text decode、capture/reuse/sleep | SENSENOVA-1b | `pipeline_sensenova_u1.py::_decode_context` → `paged_decode.py` | +| GQA K/V、three-axis RoPE、single-token mask | SENSENOVA-1c | `sensenova_u1_transformer.py` → pipeline forward | + +## SENSENOVA-1a — U1.5 distilled LoRA 是启动期单文件、单向融合 + +- 触发：`sensenova/SenseNova-U1.5-8B-MoT` 的 distilled 8-step adapter、kohya A/B key mapping、TP load 或融合失败处理变更。 +- 强制：U1.5 共享 U1 pipeline；startup 只接受 exactly one file。以 suffix/exact first-match mapping 解析 kohya A/B，fp32 计算 delta 后一次 round；fused/packed target 依声明顺序拼接，parameter `weight_loader` 负责 TP shard。zero match fail。成功后只留 adapter-name sentinel，不留 tensors；重复 load no-op。逐 parameter progressive fuse 失败必须 abort startup、reload 才可恢复，不承诺 rollback 或 adapter switching。 +- 禁止：substring multi-match、silent no-op、保留 adapter state、将一向 fused distill 当 dynamic PEFT/unload，或外推 checkpoint/quality/perf 支持。 +- 验收：CPU 覆盖 A/B、suffix overlap、fused mapping、fp32 delta、TP loader、zero-match、retained sentinel/second load 与 partial-failure abort；真实 adapter/TP/E2E 另行证据。^[PR #6516] ^[Issue #6471] + +## SENSENOVA-1b — paged AR decode 是单序列模型本地 CUDA opt-in + +- 触发：`VLLM_OMNI_SENSENOVA_PAGED_DECODE`、AR decode cache/capture、dynamic PEFT 或 level-2 sleep。 +- 强制：literal env default `"1"` 才启用；仅 CUDA、eligible head dimension 与完整 varlen signature，其他情况 eager fallback。cache 是 one-sequence internal K/V、identity block table、device write position/static capture；fit bucket 才 reuse，growth 必须 allocate/copy/block-table 后 atomic publish bucket+generation，并 recapture；graphs 进 global pool，sleep2 clear。dynamic PEFT wrappers 禁 cross-request capture/reuse；one-way fused distill 仍 eligible。 +- 禁止：把它称为 scheduler/KV manager/prefix reuse/continuous batching；跨 request 复用 dynamic adapter graph；failed grow 后保留部分 buffers；宣称通用 paged-KV、hardware parity 或性能。 +- 验收：覆盖 env/eligibility fallback、single sequence identity table、reuse/grow generation/recapture/failed grow、sleep clear、base→LoRA→base veto；scheduler-visible runner 或 multi-sequence forward 时删除本实现并改用 manager。^[PR #6516] + +## SENSENOVA-1c — native GQA 与三轴 RoPE 必须在模型 forward 保持 + +- 触发：SenseNova attention head layout、RoPE construction 或 single-token decode mask 变更。 +- 强制：native GQA 不预 expand K/V；每 forward 一次构建/共享 three-axis RoPE embeddings；single-token decode 不注入 all-zero mask。 +- 禁止：把此模型 exception 推广给 shared attention，或用单测外推 quality/performance。 +- 验收：GQA/reference、RoPE sharing 和 maskless decode 各有 model test；真实 output parity/accelerator coverage 独立取得。^[PR #6516] From 349cf2913e8079e3c3e2f7ab4ef3089f30f505d2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:34:09 +0800 Subject: [PATCH 421/465] knowledge(vllm-omni): sync 3cddda97 (PR #6783) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 3 ++- .../configuration/rules-legacy-engine-args.md | 18 ++++++++++++++++++ .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/configuration/rules-legacy-engine-args.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 3f363bfec..e25ba623b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 477083a103bef1e2c5fab174a61c268c43e9238d + previous_audited_sha: 0288c3f56eb4a4ff410194e586498e3bc8ff8362 audited_ref: main - audited_sha: 0288c3f56eb4a4ff410194e586498e3bc8ff8362 + audited_sha: 3cddda978153f3a19ecd691d343854dd074e587d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index a7d5aa46b..b14020e0e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0288c3f56eb4a4ff410194e586498e3bc8ff8362`. The canonical machine baseline +3cddda978153f3a19ecd691d343854dd074e587d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bed255b3d..ae3822a8e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0288c3f5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3cddda97` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 @@ -40,6 +40,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] | stage full-payload transport capability、topology-owned projection 或 deploy/CLI override rejection | [stage transport capability](rules-stage-transport.md) | | diffusion Ulysses SymmMem transport flag 的 deploy/CLI/default-stage projection | [diffusion parallel transport rules](rules-diffusion-parallel-transport.md) | | diffusion attention shorthand、structured config 与 global override 的优先级和最终规范表示 | [diffusion attention precedence](rules-diffusion-attention-precedence.md) | +| legacy stage engine args 的 copy boundary、connector/default injection 或 repeated build mutation | [legacy engine-args isolation](rules-legacy-engine-args.md) | | 审计配置来源和多层加工 | [config audit](config-audit-plain-language.md) | | 设计配置归一化与 parity 验证 | [config normalization parity](config-normalization-parity.md) | | 查询 `Omni()` 初始化参数 | [Omni init args](omni-init-args.md) | diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-legacy-engine-args.md b/knowledge/repos/vllm-omni/components/configuration/rules-legacy-engine-args.md new file mode 100644 index 000000000..0b4896869 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/configuration/rules-legacy-engine-args.md @@ -0,0 +1,18 @@ +--- +title: "legacy engine args 投影隔离" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, config] +sources: ["PR #6783", vllm_omni/engine/stage_init_utils.py, tests/engine/test_build_engine_args_no_mutate.py] +confidence: high +--- + +# legacy engine args 投影隔离 + +## VOMNI-CFG-1r — legacy engine-args projection 必须递归 detach source config + +- 触发：`build_legacy_engine_args_dict`、`_to_dict(stage_config.engine_args)`、finalizer、connector 或 default injection 修改。 +- 强制：先对 `_to_dict(stage_config.engine_args)` `deepcopy`，再 pop/delete/finalize/inject；source template 和所有 nested mutable objects 不得改变，重复 build 的结果等价且 idempotent。实际 `inject_omni_kv_connector_config` 在 returned dict 的 resolved connector mutation path 也必须不能 leak 回 source。 +- 禁止：只 shallow copy、只测 top-level no-mutation，或让 legacy projection 因上一次 build 的 resolved/default/connector state 改变下一次输出。 +- 验收：覆盖 pop/finalizer/default/nested dict、two successive builds，以及 connector injection 后 source snapshot 不变。证据为 CPU regression/reviewer approval；不证明 typed projection、runtime connector success 或 hardware behavior。^[PR #6783] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 763ef02b7..be4c1c841 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0288c3f5` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 3cddda97` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0ffa029dc..0e7283b27 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0288c3f5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3cddda97` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1ab035ad3..dbb043333 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0288c3f5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3cddda97` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 19bcace06..f01e13ba2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0288c3f5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3cddda97` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 88243c6e3..94b731f1c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0288c3f5` 验证存在 +- 源码校验：以上路径均已在 `main @ 3cddda97` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3ea5accd8..34152957d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0288c3f5`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3cddda97`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 477083a1 \ - --to 0288c3f5 \ + --from 0288c3f5 \ + --to 3cddda97 \ --repo \ --mode report-only ``` From f1502d2482d934a83f76d5c4739f429ece04c58d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:36:51 +0800 Subject: [PATCH 422/465] knowledge(vllm-omni): sync 5f498485 (PR #6571) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/boogu-image/_index.md | 6 +++--- knowledge/repos/vllm-omni/models/boogu-image/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 23 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e25ba623b..04c29f2ce 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0288c3f56eb4a4ff410194e586498e3bc8ff8362 + previous_audited_sha: 3cddda978153f3a19ecd691d343854dd074e587d audited_ref: main - audited_sha: 3cddda978153f3a19ecd691d343854dd074e587d + audited_sha: 5f49848536fb7a0de5d29963097d2411f54f2d86 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index b14020e0e..c0a3e2481 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3cddda978153f3a19ecd691d343854dd074e587d`. The canonical machine baseline +5f49848536fb7a0de5d29963097d2411f54f2d86`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ae3822a8e..c7b1c6094 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3cddda97` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5f498485` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index be4c1c841..0aceb2494 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3cddda97` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 5f498485` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0e7283b27..15dfd5e02 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3cddda97` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5f498485` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index dbb043333..2f7249878 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3cddda97` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5f498485` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f01e13ba2..0f65845ba 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3cddda97` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5f498485` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 94b731f1c..7d3bf9367 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3cddda97` 验证存在 +- 源码校验：以上路径均已在 `main @ 5f498485` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/boogu-image/_index.md b/knowledge/repos/vllm-omni/models/boogu-image/_index.md index 69f81adab..6ced4cd51 100644 --- a/knowledge/repos/vllm-omni/models/boogu-image/_index.md +++ b/knowledge/repos/vllm-omni/models/boogu-image/_index.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #6701", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/diffusion/models/boogu_image/, vllm_omni/entrypoints/utils.py] +sources: ["PR #6701", "PR #6571", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/diffusion/models/boogu_image/, vllm_omni/entrypoints/utils.py] confidence: high --- @@ -20,7 +20,7 @@ confidence: high 共 5 个文件： - `vllm_omni/diffusion/models/boogu_image/` -- revision/component loader、Edit-Turbo checkpoint boundary：[rules](rules.md) +- revision/component loader、Edit-Turbo checkpoint boundary：见本页目录中的 Boogu rules。 ## 依赖的共享代码模块 @@ -51,4 +51,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | +| revision/component loader、Edit-Turbo checkpoint 或 real RoPE/platform selection | [Boogu rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/boogu-image/rules.md b/knowledge/repos/vllm-omni/models/boogu-image/rules.md index 6a7776600..e32b79607 100644 --- a/knowledge/repos/vllm-omni/models/boogu-image/rules.md +++ b/knowledge/repos/vllm-omni/models/boogu-image/rules.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #6701", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py, vllm_omni/entrypoints/utils.py, tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py, tests/entrypoints/test_utils.py, tests/e2e/online_serving/test_boogu_image_edit.py] +sources: ["PR #6701", "PR #6571", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py, vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py, vllm_omni/entrypoints/utils.py, tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py, tests/diffusion/models/boogu_image/test_boogu_image_transformer.py, tests/entrypoints/test_utils.py, tests/e2e/online_serving/test_boogu_image_edit.py] confidence: high --- @@ -19,3 +19,10 @@ confidence: high - 验收：分别覆盖 global fill 与 stage override，并断言六个 consumer 的 effective revision 一致。Edit-Turbo 文档/测试仅覆盖 `Boogu/Boogu-Image-0.1-Edit-Turbo` 的 `hotfix-1k-20260708`、4 steps、guidance 1.0； 不证明 distinct Turbo class、DMD/renoise、quality 或一般 registry claim。^[PR #6701] + +## BOOGU-1b — RoPE 必须以 real cos/sin 表示贯穿 Boogu streams + +- 触发：RoPE table/gather、stream propagation 或 platform dtype selection 变更。 +- 强制：`(cos,sin)` 各为 `B,S,D`；per-axis tables 用 `use_real`/`repeat_interleave_real` adjacent-pair frequency，分别 gather/concat cos、sin。MPS CPU-gather indices 后返回原 device；NPU/MPS/no-FP64 用 FP32，否则 FP64。adjacent pairs 在 FP32 以 `even*cos-odd*sin`/`even*sin+odd*cos` rotate 后 cast x dtype；instruction/image refinement、double/single stream 和 pipeline 全部传此表示。 +- 禁止：Ascend complex path、swapped trig/pair layout，或 bitwise/broad platform/perf claims。 +- 验收：old complex reference assert-close，加 MPS/NPU selection 和 pipeline propagation；E2E/perf 仅作者报告。^[PR #6571] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 34152957d..a7a133685 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3cddda97`（2026-09-01）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5f498485`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 0288c3f5 \ - --to 3cddda97 \ + --from 3cddda97 \ + --to 5f498485 \ --repo \ --mode report-only ``` From 21ca8de70f1bbbed9f17591ff42f0f7a624d4887 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:39:31 +0800 Subject: [PATCH 423/465] knowledge(vllm-omni): sync e51fe6ec (PR #5716) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-system-runtime.md | 25 +++++++++---------- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 10 files changed, 24 insertions(+), 25 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 04c29f2ce..207a65732 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3cddda978153f3a19ecd691d343854dd074e587d + previous_audited_sha: 5f49848536fb7a0de5d29963097d2411f54f2d86 audited_ref: main - audited_sha: 5f49848536fb7a0de5d29963097d2411f54f2d86 + audited_sha: e51fe6ec1b9a9a0e14bb1fdb296d61b6593b93c6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c0a3e2481..400cadb20 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5f49848536fb7a0de5d29963097d2411f54f2d86`. The canonical machine baseline +e51fe6ec1b9a9a0e14bb1fdb296d61b6593b93c6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c7b1c6094..5febf9a68 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5f498485` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e51fe6ec` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0aceb2494..7eb18eb8b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5f498485` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ e51fe6ec` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index dc13fd25d..c51185a86 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", "PR #6814", "PR #6563", vllm_omni/diffusion/attention/, vllm_omni/diffusion/diffusion_kv/, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/npu/platform.py, tests/diffusion/diffusion_kv/, tests/diffusion/attention/test_piecewise_attn.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", "PR #6814", "PR #6563", "PR #5716", vllm_omni/diffusion/attention/, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/attention/parallel/ring_kernels.py, vllm_omni/diffusion/diffusion_kv/, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/npu/platform.py, tests/diffusion/diffusion_kv/, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_ulysses_uaa.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] confidence: high --- @@ -102,20 +102,19 @@ confidence: high SDPA 返回 LSE 供 ring accumulation，而这些调用没有该参数。实现以 K head 数计算 factor 并 同样 repeat V，却未显式检查 `H_v == H_k`；现有测试始终令两者相等，future caller/backend 必须 保持或提前断言该 invariant，不能等 SDPA 在更深处 cryptic failure。 -- 强制：`advanced_uaa` + Hybrid Ulysses/ Ring 时，在 Ulysses all-to-all 前检查 key 和 value - 原始 head 数均可被 `ulysses_degree` 整除；否则 fail-fast。padding 出来的 zero K/V head 若交给 - Ring 再 replicate 会被当成真实 group，不能作为合法 MQA/GQA。pure Ring（Ulysses=1）仍支持 - 1 或多 KV heads；valid Hybrid positive fixture 必须使用可整除布局。 +- 强制：advanced UAA 独立验证 `K==V`、`Q%KV==0`，main/joint 都适用。Ulysses `U`、ratio `R` 下先令 + `padded_KV=ceil(KV/U)*U`，再令 `padded_Q=padded_KV*R`，协调 pad Q/K/V 后 A2A；strict Ulysses + 保持 exact divisibility。Hybrid Ring 在既有 PyTorch ring path 本地 expand compact padded KV，不能 blanket reject； + Q padding >=1.5x 时按 `(Q,KV,U,label)` warn once，说明 FLOP/temp VRAM。 - 禁止：用默认 backend 的绿测证明 SDPA 修复（安装 FA/FA3 时会绕过）；用 num_heads=3、 Ulysses=2 的 padded case 当 Hybrid 正向测试；从通信量不变推断端到端性能不变。 -- 验收：4-card test 显式设 `TORCH_SDPA`，以 single-rank baseline 对 pure Ring GQA(8Q/2KV)、 - MQA(8Q/1KV) 和既有 Ulysses+Ring/AllGather-KV 输出，BF16 容差 5e-2；CPU guard 覆盖 Hybrid - padded-MQA 拒绝，另以 4Q heads、Ulysses2×Ring2 验证 valid Hybrid。ready CI 的 L4×4 job - 已收集 `test_attention_sp.py`，但不收集 `test_ulysses_uaa.py`；PR comment 的 primary matrix - 是 pre-final head `e6857888` 在 4×B300 上 4 passed、max abs 0.015625，runtime fix 已在但 final - target 未复测。修正为 4 heads 的 Hybrid 25.30s pass 只见 final PR body，未附 hardware/SHA/log。 - 因此目标 pin 有局部数值证据和未来 L4 gate wiring，但没有附带 L4 run 产物，Hybrid - guard/positive 也未被该新 job 覆盖。^[PR #5255] +- 禁止：独立 rounding（如 28/7/U2 成 32/8 以外的 Q/KV ratio）、把 padding 当真实 head、放松 malformed + shape，或外推 broad performance。CPU/main-joint、multi-GPU parity 与 4×H800 microbench 仅为有界证据。^[PR #5716] +- 验收：CPU 覆盖 main/joint 的 valid coordinated padding 与 malformed shape；multi-GPU 以 no-SP + baseline 对 SP2 `28/7`、`6/3`、SP4 `12/3` 及 Ulysses2×Ring2 的 MHA/GQA/joint。warning threshold + 与 dedup 需单独保护。PR #5716 的 4×H800 数据只是 attention microbenchmark：aligned case 接近 + baseline，而 uneven/MQA padding overhead 可显著增加；不构成模型 E2E、质量或通用性能证明。 + 既有 pure-Ring/L4 gate 证据仍只覆盖其原始范围。^[PR #5255] ^[PR #5716] ## DIFF-4h — FlashInfer mixed-dtype plan 必须绑定完整 runtime shape 与 mask 内容 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 15dfd5e02..97cf1de5b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5f498485` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e51fe6ec` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 2f7249878..f2f69030c 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5f498485` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e51fe6ec` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0f65845ba..0120241f1 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5f498485` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e51fe6ec` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 7d3bf9367..164d2bf80 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5f498485` 验证存在 +- 源码校验：以上路径均已在 `main @ e51fe6ec` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index a7a133685..235267bd3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5f498485`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e51fe6ec`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 3cddda97 \ - --to 5f498485 \ + --from 5f498485 \ + --to e51fe6ec \ --repo \ --mode report-only ``` From 814a361c4f6e5a67594cb605b1ac4ff32ab3d58e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:42:21 +0800 Subject: [PATCH 424/465] knowledge(vllm-omni): sync 67ef0b64 (PR #6908) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/moss-tts/architecture.md | 3 +++ knowledge/repos/vllm-omni/models/moss-tts/rules.md | 10 +++++++++- 11 files changed, 24 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 207a65732..a109bfa43 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5f49848536fb7a0de5d29963097d2411f54f2d86 + previous_audited_sha: e51fe6ec1b9a9a0e14bb1fdb296d61b6593b93c6 audited_ref: main - audited_sha: e51fe6ec1b9a9a0e14bb1fdb296d61b6593b93c6 + audited_sha: 67ef0b64b94d2618aaa6c4090037c16175e0b92c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 400cadb20..00b1cb91d 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e51fe6ec1b9a9a0e14bb1fdb296d61b6593b93c6`. The canonical machine baseline +67ef0b64b94d2618aaa6c4090037c16175e0b92c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5febf9a68..2054437b4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e51fe6ec` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 67ef0b64` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7eb18eb8b..f5c824785 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e51fe6ec` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 67ef0b64` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 97cf1de5b..842e5a970 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e51fe6ec` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 67ef0b64` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f2f69030c..236d84b16 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e51fe6ec` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 67ef0b64` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0120241f1..d3a297375 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e51fe6ec` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 67ef0b64` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 164d2bf80..8259c9be4 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e51fe6ec` 验证存在 +- 源码校验：以上路径均已在 `main @ 67ef0b64` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 235267bd3..c86fab724 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e51fe6ec`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 67ef0b64`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 5f498485 \ - --to e51fe6ec \ + --from e51fe6ec \ + --to 67ef0b64 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md index ceb641c4a..e51c0366a 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/architecture.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/architecture.md @@ -9,6 +9,9 @@ sources: ["PR #5635", "PR #5235", "PR #6664", vllm_omni/model_executor/models/mo # MOSS-TTS 架构 +codec-v2 的 NPU RoPE layout conversion、kernel boundary 与 non-NPU eager path 见 +[MOSSTTS-1b](rules.md#mosstts-1b-codec-v2-npu-rope-必须显式往返-gpt-j-与-neox-pairs)。 + 事实在 `main @ be335a86` 复核;变体/deploy 速查见 [index](_index.md)。 ## 模型专有部分与共享模块的边界 diff --git a/knowledge/repos/vllm-omni/models/moss-tts/rules.md b/knowledge/repos/vllm-omni/models/moss-tts/rules.md index 8cdd23f90..ea0c58338 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #5635", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, "PR #6241", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py", "PR #6543", vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, tests/entrypoints/openai_api/test_tts_adapter.py] +sources: ["PR #5635", "PR #6908", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, "PR #6241", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py", "PR #6543", vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, tests/entrypoints/openai_api/test_tts_adapter.py] confidence: high --- @@ -17,6 +17,7 @@ confidence: high | PR 描述信号 | 规则 | 第一批源码 | |---|---|---| | codec v1/v2、`number_channels`、missing weights | `MOSSTTS-1a` | `modeling_moss_tts_codec.py::MossTTSCodecDecoder._build_codec` → Stage-1 weight loader | +| codec-v2 NPU RoPE、`npu_rotary_mul`、GPT-J pairs | `MOSSTTS-1b` | `audio_tokenizer_v2.py::apply_rope` | | `_ProjectedTransformer`、`in_proj`/`out_proj`、Identity | `MOSSTTS-1a` | `audio_tokenizer.py::_ProjectedTransformer` → codec checkpoint parameter names | | online request `seed`、adapter additional information、Nano/Local/Delay/Realtime reproducibility | `MOSSTTS-3a` | `tts_adapters/moss_tts.py::_MossTTSAdapterBase.build` → variant model seed/RNG consumer | @@ -42,6 +43,13 @@ confidence: high 共享 loader 的 dtype/config 最小读取合同见 [Model Executor rules](../../components/model-executor/rules.md)。 +## MOSSTTS-1b — codec-v2 NPU RoPE 必须显式往返 GPT-J 与 NeoX pairs + +- 触发：codec-v2 `apply_rope`、time-before-heads broadcast、NPU kernel 或 RoPE table 变更。 +- 强制：保留 GPT-J interleaved pairs。NPU 从 `freqs*ts` 建 `cos_d2`/`sin_d2`、duplicate halves至 NeoX width，将 q/k 经 view-transpose-reshape-contiguous 变为 NeoX，分别 lazy-import `torch_npu.npu_rotary_mul`，再 inverse-permute 为 contiguous GPT-J；保持 time-before-heads shape/broadcast。non-NPU 保持 eager FP32 GPT-J；NPU import/kernel failure fail closed，无 recoverable fallback。 +- 禁止：直接给 kernel interleaved/noncontiguous input、改 pairing、只转 q/k 一侧，或改变 CPU/GPU path。 +- 验收：真实 NPU parity 与 non-NPU fallback 不变；target 无自动测试，只有作者 910B2C 观察，不能外推质量/性能。^[PR #6908] + ## MOSSTTS-2a — Local talker 异步输出必须保持固定 shape 和 prefill 边界 - 触发：修改 MOSS-TTS Local talker 的 `make_omni_output`、`compute_logits`、`preprocess`、async chunk 或 CUDA Graph/async schedule。 From 7c593ab13ea910904dd79710bffdbce80a52aae3 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:44:25 +0800 Subject: [PATCH 425/465] knowledge(vllm-omni): sync b6e72f38 (PR #6890) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 1 + .../vllm-omni/ci/guides/buildkite-structure.md | 16 ++++++++-------- knowledge/repos/vllm-omni/ci/rules-l4-k8s.md | 18 ++++++++++++++++++ .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 12 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 knowledge/repos/vllm-omni/ci/rules-l4-k8s.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a109bfa43..ac7c3cf05 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e51fe6ec1b9a9a0e14bb1fdb296d61b6593b93c6 + previous_audited_sha: 67ef0b64b94d2618aaa6c4090037c16175e0b92c audited_ref: main - audited_sha: 67ef0b64b94d2618aaa6c4090037c16175e0b92c + audited_sha: b6e72f3879f52cc7d75b989890a404758f87ebf6 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 00b1cb91d..11d315c23 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -67ef0b64b94d2618aaa6c4090037c16175e0b92c`. The canonical machine baseline +b6e72f3879f52cc7d75b989890a404758f87ebf6`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 179892e17..05ec03f17 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -210,5 +210,6 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie |---|---| | 审查硬件 lane（含 MiniMax-H3 DLO DP2 ready smoke 的证据边界）、回归 fence、CI 工具供应链或 xdist/shared fixture | [CI rules](rules.md) | | AMD/ROCm timeout、quarantine、memory signal、Qwen3-TTS argv 或 Qwen3-Omni control-plane fixture | [AMD/ROCm CI rules](rules-amd.md) | +| CUDA L4 Kubernetes preset、GPU-count shard、resource/retry policy | [L4 Kubernetes CI rules](rules-l4-k8s.md) | | 查看仓库特有 CI 陷阱 | [CI guides](guides/_index.md) | | 调查历史 CI 失败 | [CI incidents](incidents/_index.md) | diff --git a/knowledge/repos/vllm-omni/ci/guides/buildkite-structure.md b/knowledge/repos/vllm-omni/ci/guides/buildkite-structure.md index 0799436b6..2ec0c121d 100644 --- a/knowledge/repos/vllm-omni/ci/guides/buildkite-structure.md +++ b/knowledge/repos/vllm-omni/ci/guides/buildkite-structure.md @@ -1,10 +1,10 @@ --- title: "Buildkite 管线结构" created: 2026-07-16 -updated: 2026-08-05 +updated: 2026-09-05 type: guide tags: [vllm-omni, ci] -sources: [".buildkite/cuda/pipeline.yml", ".buildkite/cuda/rebase-pipeline.yml", "vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh"] +sources: ["PR #6890", ".buildkite/cuda/pipeline.yml", ".buildkite/cuda/rebase-pipeline.yml", .buildkite/common/ci_mirror_hardwares.yml, .buildkite/cuda/test-ready.yml, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, "vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh"] --- # Buildkite 管线结构 @@ -33,18 +33,18 @@ rebase-agent 配置快照（@122a9468），**属运营观测、可能漂移**， 归入 `tests/e2e/features//`；新增 component 测试放在对应 `tests/{component}/`，不创建平行的旧顶层目录。 - `cuda/test-ready.yml` 的 CPU fast lanes 仍按 `tests/diffusion`、 - `tests/model_executor`、`tests/entrypoints` 和 `tests/engine` 分组；feature lane - 使用 `tests/e2e/features/custom_pipeline/` 与 `tests/e2e/features/fullduplex/`，跑在 - `gpu_1_queue`、CI docker 镜像内、`HF_HOME=/fsx/hf_cache`。 + `tests/model_executor`、`tests/entrypoints` 和 `tests/engine` 分组；L4 GPU jobs 已按 + `cards_N` 投影到 `l4_1..4` Kubernetes presets，而不是旧 `gpu_1_queue`/`gpu_4_queue` Docker lanes。 + 精确资源、retry 与 ready/nightly/weekly shard 合同见 [OMNI-CI-1e](../rules-l4-k8s.md)。 ## 运营事实（rebase-agent 观测，@122a9468） - Buildkite org：`vllm`。管线：`vllm-omni-release`（CI）、`vllm-omni-rebase` （nightly 与 main CI）；rebase 分支 `dev/vllm-align`，wheel 变体 `cu130`， 上次 rebase 的 vLLM 提交 pin `1acd67a795ebccdf9b9db7697ae9082058301657`。^[CFG-buildkite] -- 队列 →（最少卡数, 硬件约束）映射：`gpu_1_queue` →（1, any）——实际为 1×L4 机器； - `gpu_4_queue` →（4, any）——实际为 4×L4；`mithril-h100-pool` →（N, h100）—— - H100/H800（`can_run_ci_test` 把 H800 视同 H100）。^[CFG-queue-map] +- 旧 rebase-agent 快照中的 `gpu_1_queue`/`gpu_4_queue` 是历史运营映射，不能描述当前仓库 L4 + job；当前 repo-side `l4_1..4` 使用 `l4-k8s` 和 `vllm.ci/gpu-pool=l4x4`。H100/H800 的 + `mithril-h100-pool` 仍是外部运营观测，使用前必须复核。^[CFG-queue-map] ^[PR #6890] ## 相关 diff --git a/knowledge/repos/vllm-omni/ci/rules-l4-k8s.md b/knowledge/repos/vllm-omni/ci/rules-l4-k8s.md new file mode 100644 index 000000000..ec6811329 --- /dev/null +++ b/knowledge/repos/vllm-omni/ci/rules-l4-k8s.md @@ -0,0 +1,18 @@ +--- +title: "CUDA L4 Kubernetes CI 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, ci] +sources: ["PR #6890", .buildkite/common/ci_mirror_hardwares.yml, .buildkite/cuda/test-ready.yml, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, tests/buildkite/test_upload_pipeline.py] +confidence: high +--- + +# CUDA L4 Kubernetes CI 规则 + +## OMNI-CI-1e — L4 jobs 必须按卡数投影到 l4-k8s preset + +- 触发：L4 queue/preset、CUDA ready/nightly/weekly split、pod resource/retry 变更。 +- 强制：`l4_1..4` 使用 `l4-k8s`、selector `vllm.ci/gpu-pool=l4x4`；N GPU 配 `10N` CPU、`40N Gi` memory、`24N Gi` shm，保留 CI image/FSx HF cache/`HF_TOKEN`。只对 `-1`/`128`/`agent_stop`/`agent_refused` infra retry once，不对 `1`/`137`；explicit step retry 覆盖 preset。ready cards 1/2/4（attention 仅4），nightly 2/3/4，weekly affected 2/4，保持 L4 SKU selector。 +- 禁止：旧 gpu queue/docker/mount-agent claim，`not cards_1` 代替 exact cards，或从 render/screenshots 推断 hardware/runtime pass。 +- 验收：CPU preset render/collection 证明各 split nonempty；真实 L4 runtime 另验。^[PR #6890] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2054437b4..ff177c2de 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 67ef0b64` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b6e72f38` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f5c824785..ba61a0b4a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 67ef0b64` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ b6e72f38` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 842e5a970..c042a6edc 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 67ef0b64` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b6e72f38` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 236d84b16..46b4190f1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 67ef0b64` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b6e72f38` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d3a297375..11bb04b1d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 67ef0b64` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b6e72f38` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8259c9be4..af5e28abb 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 67ef0b64` 验证存在 +- 源码校验：以上路径均已在 `main @ b6e72f38` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index c86fab724..48c5efad1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 67ef0b64`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b6e72f38`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from e51fe6ec \ - --to 67ef0b64 \ + --from 67ef0b64 \ + --to b6e72f38 \ --repo \ --mode report-only ``` From 8dc1e13276f3f6740ffe0f24cb990db4d34fecac Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:48:50 +0800 Subject: [PATCH 426/465] knowledge(vllm-omni): sync 57c08d34 (PR #5861) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/sana-video/_index.md | 5 +++-- .../models/sana-video/architecture.md | 11 ++++++----- .../repos/vllm-omni/models/sana-video/rules.md | 18 ++++++++++++++++++ 12 files changed, 39 insertions(+), 19 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/sana-video/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index ac7c3cf05..2b06deb63 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 67ef0b64b94d2618aaa6c4090037c16175e0b92c + previous_audited_sha: b6e72f3879f52cc7d75b989890a404758f87ebf6 audited_ref: main - audited_sha: b6e72f3879f52cc7d75b989890a404758f87ebf6 + audited_sha: 57c08d34c359cce233990fd138a8dc9f46973e34 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 11d315c23..735a11665 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b6e72f3879f52cc7d75b989890a404758f87ebf6`. The canonical machine baseline +57c08d34c359cce233990fd138a8dc9f46973e34`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ff177c2de..49a5d3259 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b6e72f38` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 57c08d34` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ba61a0b4a..2e10d6d0f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b6e72f38` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 57c08d34` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c042a6edc..582ec8146 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b6e72f38` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 57c08d34` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 46b4190f1..b09cb416b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b6e72f38` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 57c08d34` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 11bb04b1d..0d7d87000 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b6e72f38` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 57c08d34` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index af5e28abb..502ec8124 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b6e72f38` 验证存在 +- 源码校验：以上路径均已在 `main @ 57c08d34` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 48c5efad1..5e3685255 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b6e72f38`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 57c08d34`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 67ef0b64 \ - --to b6e72f38 \ + --from b6e72f38 \ + --to 57c08d34 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/sana-video/_index.md b/knowledge/repos/vllm-omni/models/sana-video/_index.md index f8f53e8cb..90faf7792 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/_index.md +++ b/knowledge/repos/vllm-omni/models/sana-video/_index.md @@ -1,10 +1,10 @@ --- title: "SANA Video" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5508", vllm_omni/diffusion/models/sana_video/, vllm_omni/diffusion/registry.py, vllm_omni/model_extras/sana_video.py] +sources: ["PR #5508", "PR #5861", vllm_omni/diffusion/models/sana_video/, vllm_omni/diffusion/registry.py, vllm_omni/model_extras/sana_video.py] confidence: high --- @@ -71,3 +71,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| | T2V/I2V、checkpoint、VAE、adapter 或验证边界 | [architecture](architecture.md) | +| native TP/CFG topology、GLUMB/RMSNorm layout 或 I2V parallel invariant | [SANA rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/sana-video/architecture.md b/knowledge/repos/vllm-omni/models/sana-video/architecture.md index 6d1f5e81f..91f56c242 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/architecture.md +++ b/knowledge/repos/vllm-omni/models/sana-video/architecture.md @@ -1,10 +1,10 @@ --- title: "SANA Video 架构" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models, diffusion] -sources: ["PR #5508", vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, vllm_omni/diffusion/request.py] +sources: ["PR #5508", "PR #5861", vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, vllm_omni/diffusion/request.py] confidence: high --- @@ -26,9 +26,10 @@ wrapper；scheduler 保留 checkpoint 的 `DPMSolverMultistepScheduler` 配置 Diffusers-adapter 并非 native pipeline 的替代实现：它由 model class 选择 SANA-specific hooks。 特别是 checkpoint 声明的 T2V Diffusers class 在请求 I2V 时必须改装为 Diffusers I2V class，且 -shared `num_frames` 要翻译为 Diffusers 的 `frames`。当前 native 范围只由单 GPU 证明；sequence/ -tensor/CFG parallel、Cache-DiT、TeaCache 和 step execution 未验证，adapter 也不提供 native -continuous batching 或并行能力。 +shared `num_frames` 要翻译为 Diffusers 的 `frames`。native pipeline 的 TP/CFG 各仅支持 `{1,2}`， +包括 TP2×CFG2；SP、text-encoder TP、PP、HSDP 在加载前拒绝，精确 layout/CFG/I2V invariant 见 +[SANA-1a](rules.md#sana-1a-native-sana-video-只支持-tpcfg-的-12-矩阵)。Cache-DiT、TeaCache 和 +step execution 仍未验证，adapter 也不提供 native continuous batching 或并行能力。 ## 从输入到输出的主要流程 diff --git a/knowledge/repos/vllm-omni/models/sana-video/rules.md b/knowledge/repos/vllm-omni/models/sana-video/rules.md new file mode 100644 index 000000000..de6197765 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/sana-video/rules.md @@ -0,0 +1,18 @@ +--- +title: "SANA-Video 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, diffusion] +sources: ["PR #5861", vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, tests/diffusion/models/sana_video/] +confidence: high +--- + +# SANA-Video 规则 + +## SANA-1a — native SANA Video 只支持 TP/CFG 的 {1,2} 矩阵 + +- 触发：native SANA Video TP/CFG、transformer layout、I2V 或 Diffusers adapter topology 变更。 +- 强制：TP/CFG 各只 `{1,2}`，TP2×CFG2 valid；SP/text-encoder TP/PP/HSDP load 前 reject，Diffusers adapter reject native TP/CFG。TP 用 QKV column/output row、distributed RMSNorm global sumsq/local weights，GLUMB packed `[value;gate]` 每 segment slice 后 local chunk2/reduction。CFG1 batch neg/pos 后 slice；CFG2 mixin branch gather/combine FP32。uninitialized group serial，initialized broken group errors propagate；I2V first frame/mask 不变。 +- 禁止：TP>2/CFG>2、cache/offload/step/support/perf claim。 +- 验收：CPU topology/layout 与 tiny multiprocess TP/CFG matrix；A800 数字仅该环境观察。^[PR #5861] From f84f17366fe8e4d918b608eb52a89224686d8b05 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 06:58:13 +0800 Subject: [PATCH 427/465] knowledge(vllm-omni): sync c59b48b4 (PR #6909) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../environment-variable-contract.md | 9 +++-- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-attention.md | 26 +++++++++++++- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/minimax-h3/_index.md | 6 ++-- .../models/minimax-h3/rules-cache-task.md | 34 ++++++++++++------- .../vllm-omni/models/minimax-h3/rules.md | 2 +- 14 files changed, 70 insertions(+), 31 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2b06deb63..860da961b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b6e72f3879f52cc7d75b989890a404758f87ebf6 + previous_audited_sha: 57c08d34c359cce233990fd138a8dc9f46973e34 audited_ref: main - audited_sha: 57c08d34c359cce233990fd138a8dc9f46973e34 + audited_sha: c59b48b47e8359827b27f54cbdb18ba0da2836b4 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 735a11665..9e648faa7 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -57c08d34c359cce233990fd138a8dc9f46973e34`. The canonical machine baseline +c59b48b47e8359827b27f54cbdb18ba0da2836b4`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 49a5d3259..b9bb59e84 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 57c08d34` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ c59b48b4` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md index 3dd9d020a..edb10ef83 100644 --- a/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md +++ b/knowledge/repos/vllm-omni/components/configuration/environment-variable-contract.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: guide tags: [vllm-omni, config, environment] -sources: ["PR #6217", "PR #6631", "PR #5221", "PR #6516", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] +sources: ["PR #6217", "PR #6631", "PR #5221", "PR #6516", "PR #6909", vllm_omni/config/environment_variable_inventory.py, tests/config/test_environment_variables.py, collect_env.py] --- # 环境变量合同与诊断边界 @@ -22,6 +22,11 @@ sources: ["PR #6217", "PR #6631", "PR #5221", "PR #6516", vllm_omni/config/envir - `environment_variable_inventory.py` 是**分类清单，不是值解析器**：实际 consumer 继续拥有默认值、解析、优先级和读取时机。PR #5221 将 `VLLM_OMNI_EVENT_DRIVEN_ORCH` 加入 `PUBLIC_OMNI`（24 个名称）：只在 `Orchestrator` 构造时读取，默认 `0`；trim/lower 后仅 `1`、`true`、`yes`、`on` 开启，其他值保留 legacy loop。它同时选择 LLM orchestration 与 serving final-output drain，部署时必须在运行 orchestrator 的 stage-0 进程启动前设置；不是 request-level 开关，且不使 diffusion output 变为无轮询。新公共 Omni 名称必须以 `VLLM_OMNI_` 开头，旧前缀仅为兼容保留。^[PR #5221] - PR #6631 处理 #6217 合并后的 main 漂移：`VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` 是公开、实验性、仅环境变量的 float 秒数开关，默认 `600`；diffusion engine 在每次 `step_streaming` 请求路径读取它，非 float 或 `<=0` 时仅 warning-once 并回退默认值。它等待 async output，不是 `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` 的 full-payload input-coordinator 控制。`ROSVOT_SOURCE_DIR` 已随 SoulX-Singer 支持移除而从 model-specific `deprecate_remove` 清单删除。 - PR #6516 将 `VLLM_OMNI_SENSENOVA_PAGED_DECODE` 加入 model-specific `promote` inventory（33→34），不是 `PUBLIC_OMNI`（仍为 24）。consumer 默认只 literal `"1"` 启用 model-local SenseNova single-sequence paged decode；这不使它成为 scheduler、prefix-cache 或通用 paged-KV setting。^[PR #6516] +- PR #6909 将 `FASTVIDEO_VSA_SM100A` 加入 model-specific `promote` inventory；它为 FastH3 + MiniMax-H3 的 FastVideo VSA 选择 native Blackwell SM100a route，`1` 还要求兼容 source-built + `fastvideo-kernel` wheel。它不是 public setting，不能从 B300 Triton 或 B200 native observation + 推断其他模型、GPU、kernel build 或性能支持。目标快照为 model-specific 58、promote 35， + `PUBLIC_OMNI` 仍为 24。^[PR #6909] - 公开设置没有全局的 env-vs-CLI/YAML/request 优先级。按每个 consumer 的合同核实：请求可变行为应属于 request schema，稳定 model/stage 行为应属于 typed config，环境变量只适合启动、平台集成、诊断或兼容回退。import/startup 时读取的值必须在启动 CLI/导入前设置；启动后的父 shell 变化不会更新既有 worker。 - stage 的 `env`（以及兼容的 `runtime.env`）仅在启动该 stage/child process 时临时应用：键和值会字符串化，child 可继承，随后恢复父进程旧环境。因此它不能追溯改变已在此前 import 缓存的值，也不应承载 request-varying 语义。日志和诊断只输出键，绝不输出 stage `env` 的值。 - `collect_env.py` 只输出安全的公共 Omni allowlist、注册的 vLLM 名称和选择的平台前缀；显式 redact 的 `HF_TOKEN`、`HUGGINGFACE_HUB_TOKEN`、`OPENAI_API_KEY` 及常见 secret-like 名称不得输出值。为了诊断不完整/损坏的 Omni 安装，清单导入失败时 collector 回退为空清单而仍可报告既有安全信息。 @@ -31,7 +36,7 @@ sources: ["PR #6217", "PR #6631", "PR #5221", "PR #6516", vllm_omni/config/envir 1. 先沿真实 consumer 确认解析、默认、优先级与读取时机；不要把 inventory 或文档表当成运行时实现。 2. 新的稳定公开 Omni 设置同时更新分类、公开参考覆盖和 prefix gate；模型专用、benchmark、platform/external、internal 名称保持各自 owner，不因被读取而自动公开。 -3. 对 model-specific name 指定迁移目的（typed config、request scope、external、internalize 或 deprecate/remove）但不要把目的当成已经实现。目标提交的 57 个 model-specific 项为 promote 34、request scope 6、external 0、internalize 11、deprecate/remove 6；#6232 仍按模型族和 owner 跟踪迁移，尚不是现有运行时保证。该 issue 的旧 seven-item removal checklist 含已退休的 `ROSVOT_SOURCE_DIR`，不能覆盖当前 inventory。 +3. 对 model-specific name 指定迁移目的（typed config、request scope、external、internalize 或 deprecate/remove）但不要把目的当成已经实现。目标提交的 58 个 model-specific 项为 promote 35、request scope 6、external 0、internalize 11、deprecate/remove 6；#6232 仍按模型族和 owner 跟踪迁移，尚不是现有运行时保证。该 issue 的旧 seven-item removal checklist 含已退休的 `ROSVOT_SOURCE_DIR`，不能覆盖当前 inventory。 4. 变更 collector 或 stage `env` 时，以真实 worker/inheritance 与 redaction 测试验收；错误安装场景不得因可选清单导入而使诊断工具本身失效。 ## 已知证据限制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2e10d6d0f..1281c4bc8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 57c08d34` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ c59b48b4` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index b9bd2fe1d..0edf1bb89 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,12 +4,36 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] +sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", "PR #6909", docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_fastvideo_vsa.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] confidence: high --- # Diffusion attention 规则 +## DIFF-1ac — FastH3 VSA 必须保留 H3 packed document、tile 和 SP 合同 + +- 触发：修改 `FASTVIDEO_VSA` 的 MiniMax-H3 route、`vsa_h3_prefix_segments`/`VideoTokenLayout`、 + FastVideo 64-token kernel、compression gate，或 Ulysses attention all-to-all。 +- 强制：H3 metadata 表示一个 packed `[text | condition | audio | video]` document；prefix segment + RLE 的 sum 必须等于 target-video start，video grid 必须完整覆盖 query rows。prefix 以不跨 modality + 边界的 64-token tiles 切分，target video 以 `(4,4,4)` tiles 切分；prefix query dense，video query + 保留所有 prefix tiles 加 top-k video tiles。tile partition、non-pad index 与 untile 必须恢复原 row + 顺序；native SM100a path 只为 even block-partner requirement 增加 transport-only partner，并在返回前移除。 +- 强制：compression gate 跟随 Q/K/V 的 Ulysses sequence-to-head all-to-all，使 local head shard 拥有 + 完整 document；仅 local 或 pure Ulysses 可进入 H3 VSA，Ring 与 AllGather SP 必须在 admission 拒绝。 + H3 route 有意不走 generic `min_seq_len` 或 `disable_when_sp_active` fallback gates；不能把这些 knobs + 当作 H3 sparse admission 的实际保护。 +- 禁止：让 prefix tile 跨 `[text|condition|audio]` boundary、把 video top-k 施加到 prefix keys、把 + odd partner 留给输出 untile，或把 Ring/AllGather partial document 交给 block-sparse kernel。不得将 + H3 route 的 CUDA error 之后的 dense fallback 描述为 sparse execution。 +- 验收：测试覆盖 segment-pure prefix、3-D video tiling、prefix-dense/top-k map、odd partner、gate + blending、packed-padding zero restore 和 pure-Ulysses gate resharing；真实 VSA 还须固定 checkpoint、 + kernel build、GPU、topology、request shape 与 top-k。PR #6909 的 B300 Triton 与独立 B200 native + SM100a observations 仅限所报 8×GPU、USP8/Ring1、four-step、1344×768/24 FPS、top-k 64 workload， + 不构成广泛性能或 support claim；两条路均依赖外部 `fastvideo-kernel`。 +- 证据缺口：multi-request layout error 会被 `fallback_on_error` 吞掉并运行 dense SDPA；尚无 + continuous-batching proof。^[PR #6909] + ## DIFF-1f — deterministic flag 只对构造时选定的 local dense FA 生效 - 触发：修改 `--fa-deterministic`、diffusion config projection 或 local `FLASH_ATTN` dispatch。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 582ec8146..ee7b63037 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 57c08d34` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ c59b48b4` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b09cb416b..261cb06c8 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 57c08d34` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ c59b48b4` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0d7d87000..1215d18a2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 57c08d34` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ c59b48b4` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 502ec8124..f070c1261 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 57c08d34` 验证存在 +- 源码校验：以上路径均已在 `main @ c59b48b4` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5e3685255..37cde7463 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 57c08d34`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ c59b48b4`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from b6e72f38 \ - --to 57c08d34 \ + --from 57c08d34 \ + --to c59b48b4 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md index 1544d066f..fe203354c 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/_index.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/_index.md @@ -96,8 +96,10 @@ confidence: high 交互与单 H100 校准边界见 MMH3-2g。 - distilled checkpoint 的 continuous RF `base_schedule`、四步语义与 partition 隔离见 MMH3-2h；它不替换 H3 的自定义 solver。 -- FastH3 是 exact FastVideo artifact 的 startup-only t2va fusion；它不使用 Turbo/native dynamic - manager，strict artifact/request/offload gates 见 MMH3-2n。^[PR #6714] +- FastH3 `vsa-datafree` 是 exact FastVideo artifact 的 startup-only T2VA fusion：50 main-DiT + compression gates 只在 resolved self backend `FASTVIDEO_VSA` 下使用，token refiner 保持 dense； + pure Ulysses 可用，Ring/AllGather、request LoRA 与 offload 均拒绝。strict artifact/request/gate + loading 与外部 kernel 边界见 MMH3-2n，packed 64-token VSA layout 见 DIFF-1ac。^[PR #6909] - H100_2 merge lane 的 FL2VA/Ref2VA/Turbo/FastH3 四个隔离 process 只构成精确 L3 media contract，不能升级为 runtime 或性能支持声明；见 MMH3-3m。^[PR #6556] - step execution 是 H3 的模型专有 stateful path：只在 resolved attention backend 能隔离多个 packed documents 时合并 request；否则仍逐 request 前向。request state、rank-0 preparation、Cache-DiT/DLO/multi-output 排除与有界验证证据见 MMH3-2k，不能由 generic runner guard 推断 multi-rank failure synchronization。^[PR #5810] diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md index dd8941af5..7675c4e68 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules-cache-task.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", "PR #6550", "PR #6666", "PR #6714", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/fasth3.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/npu/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/entrypoints/openai/video_api_utils.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_fasth3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_native_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py, tests/entrypoints/openai_api/test_video_api_utils.py] +sources: ["PR #5703", "PR #5720", "PR #5810", "PR #5837", "PR #5840", "PR #5853", "PR #5991", "PR #6476", "PR #6550", "PR #6666", "PR #6714", "PR #6909", vllm_omni/diffusion/models/minimax_h3/batched_packing.py, vllm_omni/diffusion/models/minimax_h3/fasth3.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/npu/lora.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/entrypoints/openai/video_api_utils.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_fasth3.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_native_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_step_execution.py, tests/entrypoints/openai_api/test_video_api_utils.py] confidence: high --- @@ -155,7 +155,7 @@ Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期 matching/mismatched explicit step 和 legacy/partially-constructed fallback。仍缺真实 distilled checkpoint 的 CUDA/NPU E2E 与 teacher/student 质量阈值。^[PR #5991] -## MMH3-2n — FastH3 只接受精确 artifact 并在启动时一次性融合 +## MMH3-2n — FastH3 只接受精确 artifact，并将 VSA gate 在启动时一次性融合 - 触发：FastH3 `lora_path`、load-time fusion、four-step request 或 serving gate。 - 强制：只 claim `fastvideo-lora-v2`、`finetuned_model` 位于 `fastvideo/` 且 identity 含 `fasth3` @@ -163,17 +163,25 @@ Cache-DiT、TeaCache、distilled sigma schedule 与 Turbo LoRA 的生命周期 root 与 claimed variant 都必须唯一，否则 fail closed；普通 FastVideo/PEFT/多 shard 交还 legacy dynamic LoRA，不可误认。 - 强制：声明的 low-rank/diff/`set_weight` tensor count 必须 numeric 且与实物一致；任何 unmapped、 - duplicate/unpaired factor、shape mismatch、extra/incomplete block、VSA `set_weight` 或 fused QKV/FC1 - 不支持的 full-rank delta 都失败。fusion 在 sharding 前完成：Diffusers Q/K/V 重排为 grouped native - QKV，FC1 value-first 重排为 gate-first，再加入 low-rank `B@A` 与 full-rank `diff`/`diff_b`。成功后 - 只保留永久 fused sentinel；不能 request-switch，dynamic manager 的 list/add/remove/pin 安全返回空/false。 -- 强制：serving 只允许 `t2va`、无 per-request `lora`、exact `num_inference_steps=4`（五 boundaries/ - 四 forwards）以及默认 video/audio shift `12/3`；拒绝 CPU/layerwise/DLO offload 与 Ref2VA。 -- 禁止：不得混同 Turbo/native dynamic adapters，支持 VSA/multi-adapter/Ref2VA/offload，或声称 - quality parity、平台 portability 与通用性能收益；recipe/body 的旧 steps=5 不能覆盖 source 的 4。 -- 验收：synthetic artifact 覆盖 identity/base/root/variant、metadata counts、mapping/pairs、QKV/FC1 - reorder、full/low-rank delta、block/shape 与 unapplied patch failures；request matrix 覆盖 task、steps、 - shifts、request LoRA 和各 offload gate。CPU tests 不替代真实 artifact/hardware 质量或性能验证。^[PR #6714] + duplicate/unpaired factor、shape mismatch、extra/incomplete block 或 fused QKV/FC1 不支持的 full-rank + delta 都失败。fusion 在 sharding 前完成：Diffusers Q/K/V 重排为 grouped native QKV，FC1 value-first + 重排为 gate-first，再加入 low-rank `B@A` 与 full-rank `diff`/`diff_b`。成功后只保留永久 fused sentinel； + 不能 request-switch，dynamic manager 的 list/add/remove/pin 安全返回空/false。 +- 强制：FastH3 `vsa-datafree` artifact 的 `.set_weight` 必须精确映射到 main `transformer.` DiT 的 + `blocks.{0..49}.attn.to_gate_compress.weight`：pipeline 在 streaming weights 之前只为这 50 个 main-DiT + attention 创建 zero-initialized gate projection；token refiner 保持 dense，base checkpoint 没有同名 parameter + collision。`set_weight` 是 assign 不是 base-delta；加载后以实际 `transformer_loaded` 集合调用 + `validate_fully_applied`，保证每个 injected gate 唯一且确实被 transformer 消费。 +- 强制：VSA artifact 的 main-DiT self attention resolved backend 必须为 `FASTVIDEO_VSA`（含 `self` + per-role override）；dense substitute 不可运行该 sparse student。serving 只允许 `t2va`、无 per-request + `lora`、exact `num_inference_steps=4`（五 boundaries/四 forwards）以及默认 video/audio shift `12/3`； + 拒绝 CPU/layerwise/DLO offload 与 Ref2VA。 +- 禁止：不得混同 Turbo/native dynamic adapters，支持 VSA multi-adapter、Ref2VA/offload，或声称 quality + parity、平台 portability 与通用性能收益；recipe/body 的旧 steps=5 不能覆盖 source 的 4。 +- 验收：synthetic artifact 覆盖 identity/base/root/variant、metadata counts、mapping/pairs、QKV/FC1 reorder、 + full/low-rank delta、50 个 gate 的 mapping/unique/consumption、block/shape 与 unapplied patch failures； + request matrix 覆盖 backend、task、steps、shifts、request LoRA 和各 offload gate。CPU tests 不替代真实 + artifact/hardware 质量或性能验证。^[PR #6714] ^[PR #6909] ## MMH3-2j — Turbo LoRA 只接受精确发布 artifact，并与 H3 task、sampling 和 offload lifecycle 共同门禁 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 5e88dbce3..3b36430a7 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -38,7 +38,7 @@ confidence: high | distilled、DMD2、`base_schedule`、4-step、sigma boundary | [MMH3-2h](rules-cache-task.md#mmh3-2h-distilled-sigma-schedule-按-partition-所有且以区间计步) | partition model metadata → `DMD2SigmaSchedule` → H3 video/audio shifted sigmas | | LightX2V Turbo、legacy dynamic LoRA、packed QKV/FC1、five sigma points、DLO resident A/B sidecars | [MMH3-2j](rules-cache-task.md#mmh3-2j-turbo-lora-只接受精确发布-artifact并与-h3-tasksampling-和-offload-lifecycle-共同门禁) | `lora.py::load_minimax_h3_turbo_lora` → pipeline hook/sampling validation → `DiffusionLoRAManager` | | FlashGen native LoRA、grouped QKV、adapter sigma schedule、legacy manager binding | [MMH3-2m](rules-cache-task.md#mmh3-2m-flashgen-native-lora-必须保持-artifactschedule-与-legacy-manager-边界) | `npu/lora.py::load_minimax_h3_native_lora` → H3 pipeline active-adapter validation → unchanged `DiffusionLoRAManager` ^[PR #6666] | -| FastH3、fastvideo-lora-v2、four-step、load-time fusion | [MMH3-2n](rules-cache-task.md#mmh3-2n-fasth3-只接受精确-artifact-并在启动时一次性融合) | `fasth3.py` identity/strict fusion → pipeline serving/request gate | +| FastH3 VSA、`vsa-datafree`、`FASTVIDEO_VSA`、four-step、load-time fusion | [MMH3-2n](rules-cache-task.md#mmh3-2n-fasth3-只接受精确-artifact并将-vsa-gate-在启动时一次性融合) | `fasth3.py` identity/strict fusion → pipeline gate injection/consumption → serving/request gate | | step execution、continuous batching、multi-document packing、rank-0 prepare failure | [MMH3-2k](rules-cache-task.md#mmh3-2k-h3-step-execution-必须保持请求状态attention-文档和-rank-0-prepare-隔离) | `pipeline_minimax_h3.py::{prepare_encode,denoise_step,step_scheduler}` → `batched_packing.py` → runner | | DLO、TP-local、resident layers、allocator cache、encoder/VAE staging | [MMH3-3a](rules-deployment.md#mmh3-3a-h3-dlo-必须保持-loader-layout-与-component-stage-配对) | H3 `_offload_plan` → `BoundedAllocatorCache`/`PinnedModuleStager` → pipeline encode/denoise/decode stage contexts | | RTX 5090/4090、24/32 GiB、consumer profile | [MMH3-3b](rules-deployment.md#mmh3-3b-consumer-gpu-profile-是有边界的容量证据) | recipe measurement commit/run record → exact target topology → quality/capacity validation | From 805d60d10da7c81e12f7329afcdfaa572958eb8f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:03:49 +0800 Subject: [PATCH 428/465] knowledge(vllm-omni): sync 5d20f6b9 (PR #6724) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-attention.md | 47 ++++++++------ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +- .../vllm-omni/models/minimax-h3/rules.md | 63 +++++++------------ 11 files changed, 64 insertions(+), 70 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 860da961b..92734334e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 57c08d34c359cce233990fd138a8dc9f46973e34 + previous_audited_sha: c59b48b47e8359827b27f54cbdb18ba0da2836b4 audited_ref: main - audited_sha: c59b48b47e8359827b27f54cbdb18ba0da2836b4 + audited_sha: 5d20f6b900dfcf22b42068e28a3cbbdb5b4707e7 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9e648faa7..7f21e5641 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -c59b48b47e8359827b27f54cbdb18ba0da2836b4`. The canonical machine baseline +5d20f6b900dfcf22b42068e28a3cbbdb5b4707e7`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b9bb59e84..1f06cf4ad 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ c59b48b4` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 5d20f6b9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1281c4bc8..9ace148bf 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ c59b48b4` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 5d20f6b9` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 0edf1bb89..2618ce82f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", "PR #6909", docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_fastvideo_vsa.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] +sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", "PR #6724", "PR #6909", docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_fastvideo_vsa.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] confidence: high --- @@ -80,26 +80,35 @@ confidence: high 没有证明 third-party registry override 或未继承 base class 的 backend 完成了 capability census；这些实现仍须继承合同或显式实现 method。^[PR #5891] ^[PR #5997] -## DIFF-1m — TRTLLM packed-padding 快路径必须隔离 producer contract、ragged documents 与 SAGE gate +## DIFF-1m — TRTLLM packed-padding、Skip-Softmax 与 SAGE 必须保留 producer、SP 和 schedule 合同 - 触发：修改 `PackedPaddingMetadata`、TRTLLM packed `cu_seqlens`、`supports_packed_mask_free`、 - padded-output restore、SAGE quantization gate 或 multi-document varlen capability。 -- 强制：TRTLLM 只把 `PackedPaddingMetadata` 当作 single physical batch 的 producer-owned - `[real,pad]` shortcut：Q/KV host lengths 必须是 Python int，在 flattened tensors bounds 内并与 - `max_seqlen_q/k` 及可选 `valid_kv_length` 相等；canonical two-element `int32` cu_seqlens 必须和 - Q/K device 一致。该 path 裁 Q/K/V 后把 output 补零回物理 shape，且从不读取或 materialize - `attn_mask`。generic complete cu_seqlens 仍支持 genuine multi-document ragged varlen，并删除 - paired trailing empty document；两条合同不能互相降级或混用。 -- 禁止：接受任何 nonempty TRTLLM `attn_mask`、用 producer shortcut 表示 arbitrary mask 或 - multi-request block-diagonal layout，或让未验证的 CUDA scalar read 回到 single-request fast path。 - 不得把 tail document 短于 `k_block_size` 造成的 dense fallback 描述为 SAGE 的真实-document - 检查已经完成。 -- 验收：backend tests 分别覆盖 malformed host/device/dtype/shape/max-length metadata、trim/zero - restore、generic unequal Q/KV ragged batch、trailing empty pair、complete packed metadata 加 - nonempty mask 的 rejection，以及 real multi-document output 对逐 document reference。SAGE 还要覆盖 - alignment tail 不参与 real-document minimum；该 fix 在 PR #6542 尚未提交，现存行为可整 batch - fallback dense。性能只可引用该 PR 的 exact 4×B300 Nsight all-to-all→FMHA interval，不能以 CPU mock、 - kernel gap 或 frame hash 声称 E2E latency/quality。^[PR #6542] + padded-output restore、SAGE/Skip-Softmax gate、calibration 或 multi-document varlen capability。 +- 强制：`PackedPaddingMetadata` 只表示 single physical batch producer-owned `[real,pad]`：Q/KV host + lengths 是 Python int、在 flattened bounds 内并与 max lengths/可选 `valid_kv_length` 相等；canonical + two-element `int32` cu_seqlens 与 Q/K 同 device。该 path trim Q/K/V、zero-restore output，且不读 + `attn_mask`；generic complete cu_seqlens 仍是 genuine multi-document ragged varlen，paired trailing + empty document 删除，二者不可混用。 +- 强制：TRTLLM 仅在无 SP 或 pure Ulysses 运行；Ring+`skip_softmax` 报错、Ring 的 `quant` 不使用， + AllGather-KV 拒绝。SAGE 和 Skip-Softmax 都须显式 opt-in。`threshold` 与 `target_sparsity` 互斥： + kernel `lambda = threshold_scale_factor / max_kv_len`；direct path 的 factor 是 + `threshold * max_kv_len`，故 `lambda = threshold`；calibrated path 是 + `threshold_scale_factor = a * exp(b * target_sparsity)`，故 + `lambda = a * exp(b * target_sparsity) / max_kv_len`。请求 `target_sparsity` 而无 calibration + metadata 必须失败，不得静默变 dense。 +- 强制：Skip-Softmax 的 tile 只有当 tile 内每个 query row 都满足 + `exp(tile_max[i] - running_max[i]) < lambda` 时才跳过 Softmax/PV；`QK^T` 仍执行。正的 + `disabled_until_timestep` 以已归一化、递减的 `t` gate：`t > cutoff` dense、`t <= cutoff` 才可 skip； + `0` 关闭 gate，缺 `t` 时保持 dense。MiniMax-H3 默认 flow shift `s=12` 的 + `t = s*u / (1 + (s - 1)*u)`，50 sigma points/49 denoiser forwards 下 cutoff `.99`/`.97`/`.95` + 分别留下 6/14/19 个 dense forwards；这些 counts 只属于该 schedule。 +- 禁止：接受 nonempty TRTLLM `attn_mask`、把 shortcut 用作 arbitrary mask/multi-request layout，或把 + tail document 短于 `k_block_size` 的 dense fallback 当作 real-document SAGE gate 完成。不得将本 + docs-only PR #6724 外推为 runtime、数值、质量或性能证明。 +- 验收：覆盖 malformed metadata、trim/zero restore、unequal Q/KV ragged、trailing empty、complete + metadata+mask rejection、real-document SAGE minimum；再覆盖 SP admission、mutually-exclusive + controls、missing calibration fail、tile-wide predicate、timestep gate/missing `t` 和 H3 flow-shift + cutoff counts。^[PR #6542] ^[PR #6724] ## DIFF-1h — RainFusion irregular tail 由 MindIE-SD 保护，不能 padding 伪对齐 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ee7b63037..b537ac90d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ c59b48b4` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 5d20f6b9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 261cb06c8..11ef8d0dc 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ c59b48b4` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 5d20f6b9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 1215d18a2..16bdc33be 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ c59b48b4` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 5d20f6b9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f070c1261..a5fc7f11a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ c59b48b4` 验证存在 +- 源码校验：以上路径均已在 `main @ 5d20f6b9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 37cde7463..1d4b9f79f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ c59b48b4`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 5d20f6b9`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 57c08d34 \ - --to c59b48b4 \ + --from c59b48b4 \ + --to 5d20f6b9 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index 3b36430a7..f35b614c0 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", "PR #6040", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/user_guide/diffusion/attention_backends.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526", "PR #6410", vllm_omni/diffusion/models/minimax_h3/denoise_loop.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", "PR #6040", "PR #6724", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526", "PR #6410", vllm_omni/diffusion/models/minimax_h3/denoise_loop.py] confidence: high --- @@ -102,44 +102,29 @@ confidence: high 当前自动测试没有 MUSA dispatch/parity，CPU reference 只验证 96 维旋转和 32 维 passthrough。^[PR #5881] -## MMH3-1e — H3 TRTLLM 必须从 packed 结构裁掉 padding 并隔离短序列 role - -- 触发：修改 H3 的 TRTLLM default、packed `cu_seqlens`/mask、SAGE quant、Skip-Softmax、 - token-refiner role 或 denoise timestep context。 -- 强制：H3 只在支持的 datacenter Blackwell SM100/103、head_dim=128、FlashInfer kernel 可用且 - 声明 compatible packed/mask-free path 时默认 dense BF16 TRTLLM；SM120/121、缺依赖、错误 - head dim 或需要任意 mask 的路径保留平台 fallback，默认不自动开启 SAGE/Skip-Softmax。 -- 强制：四个 packed metadata key 必须完整、Q/K batch/terminal 覆盖一致。单 request 的 - producer-owned `[real,pad]` path 仅在 non-Ring backend 声明 `supports_packed_mask_free()` 时可省略 - `attn_mask`，并同时发布 Python `q_length`/`kv_length` 与同 device、`int32`、canonical - `[0,length]` 的 `PackedPaddingMetadata` views；TRTLLM 必须核对长度、bounds、max-length 与 - `valid_kv_length` 后裁掉 suffix、再 quantize/attention，并把输出补零到 Ulysses 物理 shape。 - TRTLLM 对任何 nonempty `attn_mask` 一律 fail closed；generic ragged cu_seqlens 仍是多 document - block-diagonal path，`[0,N,N]` 尾部空 sequence 必须折叠。AllGather-KV 的 local-Q/global-KV - metadata 不对称合同在初始化时明确拒绝，真正支持按 review 明确延期。 -- 强制：main DiT 与 `minimax_h3.token_refiner` 使用独立 role；任一 KV sequence 短于 SAGE - `k_block_size` 时该 input 告警并走 dense TRTLLM，不能让 14-token refiner 产生 non-finite， - 也不能因此关闭 main DiT 的 SAGE。H3 将 scheduler 的降序 sigma（1→0）发布为 - `normalized_timestep`；Skip-Softmax 在 sigma 大于 `disabled_until_timestep` 时保持 dense， - 到达或低于阈值才启用。 -- 禁止：把 producer-owned packed-padding shortcut 泛化为任意 mask、multi-request batch 或第三方 - metadata；让 padding 进入 SAGE block quantization；把 combined packed-H3 优化视频归因于本 PR - 单一改动。请求级 `set_forward_context` wrapper 会在 `finally` 恢复 prior context，因此 loop 内的 - step mutation 不会泄漏到后续请求。 -- 验收：测试覆盖 generic ragged batch、single-request packed-padding suffix trim/zero restore、empty - terminal、complete packed metadata 加 nonempty mask 的 unconditional rejection、invalid metadata、 - short-role dense fallback、AllGather rejection 与 non-finite regression。Continuous batching 必须 - 覆盖 aligned non-final request 的严格递增 `cu_seqlens`；SAGE gate 应只依据 real documents，当前 - multi-request alignment-tail case 仍可能使整 batch dense，是 PR #6542 留下的 unresolved P3，不能 - 声称 SAGE 保持启用。PR 报告的 4×B300 SXM6、TP1/Ulysses4/Ring1、1344×768、243 frames、49 - updates、dense BF16 TRTLLM、单 warmup 后 Nsight interval 从 final Ulysses all-to-all 到 main FMHA： - 9,797/9,680 paired samples的 gap p50 499.683→0.352 us、p95 698.308→0.384 us，且中间 GPU - activity/D2H copy 为 244,925/97,970→0/0；这只证明该 exact profiling interval，不是 E2E - latency、quality 或其他 workload/backend/topology 的保证。此前 PR 的 4×B300 SM103、 - 1248×768、209 frames、50 steps FA4/TRTLLM 对比仅绑定该 prompt/seed/topology：PSNR 27.10、 - SSIM 0.8880 不是质量 gate；83.854→71.990 s diffusion、88.558→76.176 s wall 的 warmed A/B - 来自 review follow-up 外部分支 commit `20cc23ae`，其 artifacts 明确是 combined packed-H3 - optimization evidence，并非目标 commit 的隔离实验，不能泛化为稳定 14% 保证。^[PR #5779] ^[PR #6542] +## MMH3-1e — H3 TRTLLM 必须以兼容性、packed 合同和 role 隔离决定 dense/opt-in 路径 + +- 触发：修改 H3 的 TRTLLM default、packed `cu_seqlens`/mask、SAGE、Skip-Softmax、 + `minimax_h3.token_refiner` role 或 denoise timestep context。 +- 强制：仅兼容 datacenter Blackwell SM100/103、`head_dim=128`、有 FlashInfer trtllm-gen capability + 且模型声明 packed 或 mask-free path 时，H3 才自动选 dense BF16 `TRTLLM_ATTN`。workstation + Blackwell、缺依赖、错误 geometry 或需要 mask 的路径必须走平台 fallback；显式不兼容选择必须启动 + 失败。SAGE 与 Skip-Softmax 都是默认关闭的有损 opt-in。 +- 强制：TRTLLM 只允许无 SP 或 pure Ulysses；Ring 与 `skip_softmax` 组合报错、Ring 的 `quant` + 不生效，AllGather-KV 在选择时拒绝。main DiT 与 token refiner 必须是独立 role；`per_role` 的 + `AttentionSpec` 替换整个 default spec，故 token refiner 要显式只配 dense backend，不能继承 SAGE + 或 Skip-Softmax。H3 checkpoint 没有 calibration metadata，recipe 应使用 direct + `threshold=0.05`，而非 `target_sparsity`。 +- 强制：单请求 producer-owned `[real,pad]` metadata 必须完整、Python host length 与同-device + `int32` canonical cu views 一致且在 bounds；TRTLLM 裁 suffix 后恢复零输出，任何 nonempty + `attn_mask` fail closed。generic complete cu_seqlens 保留真实 multi-document ragged path，尾部空 + pair 折叠；不可把两种合同互换。短于 SAGE `k_block_size` 的实 KV 保持 dense，不得因 token refiner + 关闭 main DiT。 +- 禁止：把 packed shortcut 泛化为 arbitrary mask、multi-request layout 或第三方 metadata；让 padding + 进入 SAGE quantization；将 docs-only PR #6724 当作 runtime、质量或性能证明。 +- 验收：覆盖 default/fallback/explicit-incompatible matrix、pure-Ulysses/Ring/AllGather admission、 + whole-spec `per_role` replacement 和 H3 direct-threshold recipe；并保留 packed trim/zero restore、 + malformed metadata、ragged/empty terminal、mask rejection和 short-role dense regression。^[PR #6542] ^[PR #6724] ## MMH3-1g — H3 NPU mask-free opt-in 必须保留 Ring 与 malformed-metadata fallback From 880542b085e0715fa2bb5328070f1582014795c3 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:07:59 +0800 Subject: [PATCH 429/465] knowledge(vllm-omni): sync d5fd9eb6 (PR #6953) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-output-lifecycle.md | 13 ++++++++++--- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/sana-video/_index.md | 7 ++++++- .../vllm-omni/models/sana-video/architecture.md | 10 +++++++++- 12 files changed, 37 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 92734334e..e1705ca3e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: c59b48b47e8359827b27f54cbdb18ba0da2836b4 + previous_audited_sha: 5d20f6b900dfcf22b42068e28a3cbbdb5b4707e7 audited_ref: main - audited_sha: 5d20f6b900dfcf22b42068e28a3cbbdb5b4707e7 + audited_sha: d5fd9eb6bbeb067582a42f7bd060996c7c6931c0 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7f21e5641..43ef1df5f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -5d20f6b900dfcf22b42068e28a3cbbdb5b4707e7`. The canonical machine baseline +d5fd9eb6bbeb067582a42f7bd060996c7c6931c0`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1f06cf4ad..1304009e9 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 5d20f6b9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d5fd9eb6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9ace148bf..6c861eaf0 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: [] # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 5d20f6b9` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ d5fd9eb6` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 3a2000cf8..421f47980 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py] +sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", "PR #6953", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] confidence: high --- @@ -130,9 +130,16 @@ confidence: high ## DIFF-1s — diffusion output type 必须从模型声明闭合到 topology 与 formatter - 触发：diffusion pipeline 产生 image/video/audio/action 等输出，或修改 model metadata、registry alias、topology `final_output_type`、formatter/post-process 与 multiprocess 传递路径。 -- 强制：model class 与 registry alias 解析到同一 canonical output metadata；默认单 stage config、multi-stage final stage、video endpoint capability 与 `OmniRequestOutput` formatter 必须传播同一类型。未知模型才回 image；非媒体输出使用稳定的 `DiffusionOutput.output` key，并注册可跨 orchestrator multiprocess 边界解析的 module-level post-process callable。 +- 强制：metadata 按 direct class → metadata alias → registry architecture class/alias → default + 的顺序解析；model class 与 registry alias 必须解析到同一 canonical output metadata。默认单 + stage config、multi-stage final stage、video endpoint capability 与 `OmniRequestOutput` formatter + 必须传播同一类型。未知模型才回 image；非媒体输出使用稳定的 `DiffusionOutput.output` key，并注册 + 可跨 orchestrator multiprocess 边界解析的 module-level post-process callable。SANA 的 + `SanaVideoPipeline` 与 `SanaImageToVideoPipeline` 都必须 direct metadata 声明 + `final_output_type="video"`；这是 serving metadata 分类，非 I2V input schema、denoise、topology + 或 VAE 合同变更。 - 禁止：只在 topology 标 video 但 formatter 仍发 image；要求每个 stage 都是 diffusion 才允许 video final；遗漏已注册 video alias 后静默回 image；用 ndarray 形状猜语义，或把局部 closure 作为跨进程 hook。 -- 验收：逐个 canonical class/alias 断言 default 与 multi-stage final type，video/image endpoint 正负 capability 及最终 output type；action 另验证 post-process callable 可跨目标进程边界使用，最终 output 保留 stable key、shape、dtype 和有限性。^[PR #4222] ^[PR #5885] +- 验收：逐个 canonical class/alias 断言 default 与 multi-stage final type，video/image endpoint 正负 capability 及最终 output type；action 另验证 post-process callable 可跨目标进程边界使用，最终 output 保留 stable key、shape、dtype 和有限性。PR #6953 的 CPU metadata unit test 只断言两种 SANA class 均分类为 video；PR 所述 online/offline 行为不是新的 endpoint 或真实模型 E2E 证据。^[PR #4222] ^[PR #5885] ^[PR #6953] ## DIFF-9a — 共享 PyAV 预构造帧 mux 必须闭合资源与音频时间零点 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index b537ac90d..9e87b4bcc 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 5d20f6b9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d5fd9eb6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 11ef8d0dc..7367b071d 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 5d20f6b9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d5fd9eb6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 16bdc33be..b8c1db327 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 5d20f6b9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d5fd9eb6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a5fc7f11a..447506aa6 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 5d20f6b9` 验证存在 +- 源码校验：以上路径均已在 `main @ d5fd9eb6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1d4b9f79f..6cfa53ac0 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 5d20f6b9`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d5fd9eb6`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from c59b48b4 \ - --to 5d20f6b9 \ + --from 5d20f6b9 \ + --to d5fd9eb6 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/sana-video/_index.md b/knowledge/repos/vllm-omni/models/sana-video/_index.md index 90faf7792..b2d758bf6 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/_index.md +++ b/knowledge/repos/vllm-omni/models/sana-video/_index.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5508", "PR #5861", vllm_omni/diffusion/models/sana_video/, vllm_omni/diffusion/registry.py, vllm_omni/model_extras/sana_video.py] +sources: ["PR #5508", "PR #5861", "PR #6953", vllm_omni/diffusion/models/sana_video/, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/model_extras/sana_video.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] confidence: high --- @@ -14,6 +14,8 @@ confidence: high - 知识树 owner：`models/sana-video`；上游目录 `sana_video`。 - registry 登记的 architecture / stage key：`SanaImageToVideoPipeline`；`SanaVideoPipeline`。 +- 两者都在 direct diffusion metadata 中声明 `final_output_type="video"`，使 serving 将 final + stage 分类为 video；这不定义或改变 I2V 请求输入、denoise、native/adapter topology 或 VAE 行为。 ## 源码路径 @@ -55,6 +57,9 @@ confidence: high - 先运行 `tests/diffusion/models/sana_video/` 的组件回归；native/adapter、T2V/I2V、480p/720p 的真实权重功能覆盖在 L4 serving expansion。golden 与 stage-alignment 是按需 correctness 工具，不是默认 per-PR gate。 +- PR #6953 另有 `tests/entrypoints/openai_api/test_video_pipeline_capability.py` 的 CPU metadata + unit：只验证两种 SANA pipeline 名称的 serving output 分类为 video；PR 报告的 online/offline + 情形不是新增 endpoint 或真实模型 E2E。 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/sana-video/architecture.md b/knowledge/repos/vllm-omni/models/sana-video/architecture.md index 91f56c242..69f1c4375 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/architecture.md +++ b/knowledge/repos/vllm-omni/models/sana-video/architecture.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: architecture tags: [vllm-omni, models, diffusion] -sources: ["PR #5508", "PR #5861", vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, vllm_omni/diffusion/request.py] +sources: ["PR #5508", "PR #5861", "PR #6953", vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, vllm_omni/diffusion/request.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] confidence: high --- @@ -18,6 +18,10 @@ self-attention 是模型专有 ReLU-kernel linear attention；prompt cross-atten `vllm_omni.diffusion.attention.Attention`。VAE、component discovery、offload、request batch、 postprocess 与 worker 生命周期仍属于共享 diffusion owner。 +Serving 只通过 direct model metadata 将上述两个 pipeline 的 final output 分类为 `video`。 +共享 resolver 的优先级是 direct class、metadata alias、registry architecture class/alias、default； +未知项仍为 image。该分类不提供或校验 I2V input schema，也不改变 denoise、topology 或 VAE。 + ## 配置、checkpoint 与兼容范围 两个 released 2B checkpoint 由 `sample_size` 决定默认 profile：30 对应 480×832，22 对应 @@ -50,3 +54,7 @@ conditioning mask 令该首帧在 denoise 中固定，同时使用 spatial times 硬件前提，不能把收集或跳过当作完整功能验证。480p T2V 的 frozen golden 只使用 case-specific 0.91 SSIM gate；其余三例保持 0.93。该差异从 native prompt cross-attention 的 vLLM-Omni Attention 与 reference SDPA 边界开始，不能据此放宽其他 case。^[PR #5508] + +PR #6953 的 `test_video_pipeline_capability.py` 是 CPU metadata unit test：它仅断言两种 SANA +class 的最终 serving 分类为 video；PR 叙述的 online/offline 行为不构成新增 endpoint 或真实模型 +E2E 覆盖。^[PR #6953] From 88dadbdb5e5e764ae5c59a303d4722db06d391a3 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:13:05 +0800 Subject: [PATCH 430/465] knowledge(vllm-omni): sync 95a93116 (PR #5543) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 5 ++++ .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 6 ++--- .../components/diffusion/rules-attention.md | 26 ++++++++++++++----- .../diffusion/rules-system-runtime.md | 21 +++++++-------- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 12 files changed, 49 insertions(+), 31 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e1705ca3e..f91f0005a 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 5d20f6b900dfcf22b42068e28a3cbbdb5b4707e7 + previous_audited_sha: d5fd9eb6bbeb067582a42f7bd060996c7c6931c0 audited_ref: main - audited_sha: d5fd9eb6bbeb067582a42f7bd060996c7c6931c0 + audited_sha: 95a93116960e364cc759c49023a4c35284157235 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 43ef1df5f..e4ab6cbda 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d5fd9eb6bbeb067582a42f7bd060996c7c6931c0`. The canonical machine baseline +95a93116960e364cc759c49023a4c35284157235`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index cf849e371..2654529d4 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -32,6 +32,11 @@ confidence: high - 触发：增加模型、硬件路径、nightly/ready/merge lane，或升级运行时/镜像。 - 强制：硬件 helper 从 `pyproject.toml` 派生 platform/SKU/`cards_N`，default 为 `cards_1`；Buildkite selector 按 cards count 切分，直接写 SKU marker 被 pre-commit 拒绝。^[PR #6650] +- 强制：CUDA `MIRROR_HW` 只接受 empty/unset 或不分大小写的 `b200`，否则 upload fail；NPU 忽略它。 + 省略 preset 时从 `-m` 推导：unset 匹配 H100/L4（两者都有时取 H100），b200 只匹配显式 B200； + 多个正 `cards_N` 取最大值，正 marker 优先于 negation，只有负 card marker 时取该 chip 最大已有 + preset，匹配 chip 却没有任何 card marker 才 fail。显式 preset 优先；b200 跳过 H100/L4 preset， + 不匹配 B200 marker 的 inferred step 也 omit，且 selector 不重写原 `-m`。^[PR #5543] - 强制：资源数、pytest marker、pipeline 路由和共享 stage registry 同步；必需依赖、模型初始化 或 collection 失败必须让 job 失败，并证明目标测试数大于零且原失败在对应硬件消失。默认 `prepend` import mode 下，whole-tree collection 遇到同名叶子目录时，所有中间父目录必须用 diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 1304009e9..d70e169ef 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d5fd9eb6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 95a93116` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 6c861eaf0..b8d2fd43e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -4,13 +4,13 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, components, diffusion] -sources: [] +sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/attention/backends/flashinfer_attn.py] --- # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d5fd9eb6` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 95a93116` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -38,7 +38,7 @@ sources: [] | Wan VAE height/width spatial reshard、empty tail、attention extent | [Wan spatial-shard rules](rules-wan-spatial-shard.md) | | multi-DiT、dotted `_dit_modules`、loader-to-offloader handoff 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | -| local FlashAttention deterministic opt-in、role-aware masked cross-attention、NPU packed mask-free/laser fallback、TRTLLM packed-padding 与 config propagation | [attention rules](rules-attention.md) | +| Blackwell FA4、显式/auto CUDA backend、per-role mask/SP pad；local FlashAttention deterministic opt-in、role-aware masked cross-attention、NPU packed mask-free/laser fallback、TRTLLM packed-padding 与 config propagation | [attention rules](rules-attention.md) | | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | | video/audio mux、DLO DP wave、result queue、async pump、SHM ownership、async-output wait 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 2618ce82f..226be8314 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -4,12 +4,27 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", "PR #6724", "PR #6909", docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_fastvideo_vsa.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] +sources: ["PR #5543", "PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", "PR #6724", "PR #6909", docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/cuda/platform.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_fastvideo_vsa.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py"] confidence: high --- # Diffusion attention 规则 +## DIFF-1ad — CUDA backend 选择必须区分显式承诺与自动回退 + +- 触发：修改 CUDA platform selector、`FLASH_ATTN`/`FLASHINFER_ATTN`/`CUDNN_ATTN`/TRTLLM、SP + auto-pad 或 paged dispatch。 +- 强制：显式 backend 必须运行或以可操作错误失败，auto 才可回退。Blackwell 的 `FLASH_ATTN` 仅可用 + `flash_attn.cute` FA4；显式缺 FA4 报错，auto 继续选择。cuDNN、FlashInfer、Sage、TRTLLM 都须按 + 实际 head-size/所需 symbol 验证（SM10x TRTLLM probe 真正 API），不得以 import 代替能力。 +- 强制：mask/SP pad 按已解析的 per-role `AttentionSpec` 能力判断，FlashInfer CuTe 不接受 custom + mask；未知 `head_size=-1` 的 capability query 不得触发大小校验。formal paged 走 native backend， + profiling 才可 dense/SDPA fallback。 +- 禁止：显式 backend 在不兼容时静默替换为 MATH/SDPA/native；从相等维度猜 BHS/BSH layout，或把 + import 成功、CPU contract test、早期 B300 smoke 外推为跨硬件功能、质量或性能支持。 +- 验收：覆盖 Blackwell FA4 缺失与可用、显式失败/auto fallback、CuTe mask、per-role SP pad、未知 + head sentinel、真实 wrapper/TRTLLM symbol 及无 FA4 的 paged path。^[PR #5543] + ## DIFF-1ac — FastH3 VSA 必须保留 H3 packed document、tile 和 SP 合同 - 触发：修改 `FASTVIDEO_VSA` 的 MiniMax-H3 route、`vsa_h3_prefix_segments`/`VideoTokenLayout`、 @@ -161,12 +176,11 @@ confidence: high - 禁止：以 `Q == K` 推断 self-attention，或把 key-mask indices 用于 query gather/scatter；这会丢弃 query rows，且在 batch 大于一时可能令 query 对齐到另一样本的 K/V。 - 验收：CPU/mock 覆盖 self 与 `role="cross"` 的 masked varlen routing（包括 Q/K 等长），精确断言 cross 的 dense Q offsets 与 K/V offsets；GPU 将不同 batch valid-K lengths 的 cross output 与 SDPA 对照。^[PR #5866] -## DIFF-1n — cuDNN attention 的单 token 形状必须提前走 MATH +## DIFF-1n — cuDNN 单 token 仅限 auto MATH workaround -- 触发：修改 cuDNN diffusion attention dispatch，或引入可能出现单 token 的动态 Q/K 序列长度。 -- 强制：当 query 或 key 的序列长度为 1 时，在进入 cuDNN-only context 前显式使用 `sdpa_kernel([SDPBackend.MATH])`；其他形状才使用显式 cuDNN，并保留符号 shape 被 cuDNN 拒绝时的运行时 fallback。 -- 禁止：把单 token 形状交给 cuDNN 后等待 eager exception；用包含 FLASH/MATH 的 priority list 掩盖 backend 选择；因 Q/K 长度不同而丢弃已有 attention mask。 -- 验收：CPU/mock 参数化覆盖 `(1,1)`、`(8,1)`、`(1,8)` 与 `(8,8)`，精确断言选择的 SDP backend；另覆盖 compile、masked unequal-length 与 unmasked native path，并说明真实 GPU 数值仍需独立验证。^[PR #6070] +- 触发：修改 cuDNN dispatch 或动态 Q/K 长度。auto 的 Q/K=1 可在 cuDNN 前走 `SDPBackend.MATH`； + 显式 `CUDNN_ATTN` 必须拒绝该形状，不能静默换 backend。覆盖 `(1,1)`、`(8,1)`、`(1,8)`、`(8,8)` + 的 explicit/auto dispatch、compile 和 mask。^[PR #6070] ^[PR #5543] ## DIFF-1z — 混合 causal/full attention 必须经过共享 Attention 能力路径 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index c51185a86..815dbfff7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -65,7 +65,8 @@ confidence: high - 触发：启用 paged scheduler、增加 diffusion attention backend 或平台 override。 - 强制：启动阶段解析并验证每层实际 backend 与当前平台；走 Omni platform hook，保留 piecewise/SP 等能力。HunyuanImage3 paged requests 已有 GPU/NPU production caller；NPU SDPA 只可作 - startup memory-profile 的 dense fallback，正式 paged request 必须走 native backend。 + startup memory-profile 的 dense fallback，正式 paged request 必须走 native backend；不能因 dense + FA4 缺失拒绝 Blackwell formal paged path。 - 禁止：未知平台继承 GPU 默认；分配 cache 后到首次 dispatch 才失败；静默占用 paged KV 却继续 dense；直接换用 upstream backend 丢失 Omni 能力。 - 验收：unsupported platform/backend fail-fast；CUDA/NPU 对应 lane 和正式模型 E2E 证明 @@ -77,7 +78,8 @@ confidence: high ## DIFF-4f — paged attention metadata 不从 padding 猜逻辑布局 - 触发：paged adapter 修改 GQA、prefix/current-token packing、LSE layout 或二维特例。 -- 强制：显式传递 Q/KV head 数、每请求 prefix/current span 和 layout；保持压缩 GQA 与 packed +- 强制：显式传递 Q/KV head 数、每请求 prefix/current span 和 layout；LSE producer 必须声明 `bhs` + 或 `bsh` 后规范化，绝不按 shape（含 `sequence == heads`）推断；保持压缩 GQA 与 packed current-token 语义，单 token/单 head 二维输入仍使用同一合同。 - 禁止：从 padding 长度推断逻辑跨度；在 `sequence == heads` 时猜 LSE 维度；fallback 静默换 backend 或把 KV heads 扩成 Q heads。 @@ -124,6 +126,9 @@ confidence: high fp8_e4m3，wrapper 输出恢复原 query dtype。auto backend 按 compute capability 选 major>=10 `cute-dsl`、major>=9 `fa3`、否则 `fa2`；non-CuTe 才接受 custom mask，CuTe nontrivial mask、 causal+explicit mask 或无法 pack 的 mask 必须回退原始输入的 Torch SDPA。 +- 强制：`flashinfer_backend` 即使未设 dtype 也必须经 `AttentionSpec.backend_kwargs()` 序列化到 impl； + platform availability 必须 probe 实际 `prefill.BatchPrefillWithRaggedKVCacheWrapper` symbol，而非仅 + 顶层 import。每层 non-CuTe 私有 128MiB workspace 的累计 OOM 风险仍未解决，不得声称已共享/懒分配。 - 强制：plan key 至少绑定 batch、Q/KV length、Q/KV heads、QK/K/VO head dim、Q/K/V/output dtype、causal、scale 与是否有 mask；shape/dtype 改变重建 indptr/plan，unmasked 同 key 可复用。 mask shape 相同也可能内容不同，所以每次 masked call 都必须重新 plan，不能只按 pointer/shape @@ -133,15 +138,9 @@ confidence: high - 禁止：把本 PR 早期 Sage/TRTLLM kernels 当成 merged scope（review 后删除并转交 TRTLLM backend）；把 activation 直接 cast 成 FP8 描述成带 scale 的 checkpoint quantization；用 `supports_attention_mask=True` 推断 CuTe 支持任意 mask。 -- 验收：mixed QK/V dtype 只能在 FlashInfer >=0.6.16rc1；目标对可解析的 <=0.6.15 fail-fast， - 但没有 dependency pin，只有 docs/code gate。该 gate 比较配置 override：两者均为 None 时立即 - 返回，即使 caller 实际传入 mixed Q/K 与 V tensor 也不拦；capability 必须最终按 plan 中真实 - q/k/v dtype 校验。缺失/非法 `flashinfer.__version__` 也会直接放行，是未封闭 gap。须补测试覆盖 - dtype allowlist、old/missing version、auto backend、device mismatch、plan reuse/replan、同 shape - 不同 mask、CuTe/causal fallback、GQA 与 output dtype；本 PR 没有新增 repo test，只有外部 smoke。 - PR 的 Cosmos3 1280×720、35 steps、seed42，B200/B300 DiT timing 与 Nano 前24帧 LPIPS=0.051496 - 只绑定 vLLM-Omni `4ce23c33` + FlashInfer `d0889c7c` 的 pre-final 环境；final config/version gate - commit 后无复测，且这些不是端到端 latency、稳定 speedup 或通用质量阈值。^[PR #5344] +- 验收：mixed QK/V 仅 FlashInfer >=0.6.16rc1；按真实 plan dtype 检验可解析的旧版、缺失/非法版本 + 都 fail-fast。覆盖 allowlist、auto、device、plan/mask replan、CuTe/causal、GQA/output dtype。 + PR #5344 的 pre-final B200/B300 smoke 不是 E2E latency、稳定 speedup 或通用质量证据。^[PR #5344] ## DIFF-4i — 模型能力 metadata 必须跨 architecture 与 pipeline 两个 key space 解析 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 9e87b4bcc..59b1fac49 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d5fd9eb6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 95a93116` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 7367b071d..e30b923c5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d5fd9eb6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 95a93116` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index b8c1db327..029784fbe 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d5fd9eb6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 95a93116` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 447506aa6..c79f7afb7 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d5fd9eb6` 验证存在 +- 源码校验：以上路径均已在 `main @ 95a93116` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6cfa53ac0..bd79b4f32 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d5fd9eb6`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 95a93116`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 5d20f6b9 \ - --to d5fd9eb6 \ + --from d5fd9eb6 \ + --to 95a93116 \ --repo \ --mode report-only ``` From 3bc52f259623924cc01e91810cbe973fe826eb76 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:17:44 +0800 Subject: [PATCH 431/465] knowledge(vllm-omni): sync 3b6ce984 (PR #5882) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-component-lifecycle.md | 11 +++++++++-- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/sana-video/_index.md | 4 ++-- .../models/sana-video/architecture.md | 10 +++++++--- .../repos/vllm-omni/models/sana-video/rules.md | 18 ++++++++++++++++-- 13 files changed, 46 insertions(+), 21 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index f91f0005a..18682c13b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d5fd9eb6bbeb067582a42f7bd060996c7c6931c0 + previous_audited_sha: 95a93116960e364cc759c49023a4c35284157235 audited_ref: main - audited_sha: 95a93116960e364cc759c49023a4c35284157235 + audited_sha: 3b6ce98421a4a32ad2c0a44e241a43f29f2ce5fa # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e4ab6cbda..548313c50 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -95a93116960e364cc759c49023a4c35284157235`. The canonical machine baseline +3b6ce98421a4a32ad2c0a44e241a43f29f2ce5fa`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index d70e169ef..554f2d634 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 95a93116` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3b6ce984` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index b8d2fd43e..79f4a5186 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 95a93116` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 3b6ce984` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md index 70472ce2c..9793522b6 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md @@ -1,10 +1,10 @@ --- title: "Diffusion component lifecycle 规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5720", "PR #5853", "PR #5884", "PR #6486", "PR #6591", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/offloader/startup.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py", "PR #6072"] +sources: ["PR #5720", "PR #5853", "PR #5882", "PR #5884", "PR #6486", "PR #6591", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/offloader/startup.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/models/sana_video/test_cache_offload.py, tests/diffusion/test_diffusion_model_runner.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py", "PR #6072"] confidence: high --- @@ -13,6 +13,13 @@ confidence: high 本页承载共享 diffusion owner 的 multi-DiT component discovery 合同；模型专有 component 名和 residency 留在模型 owner。规则入口与其他共享机制仍见 [Diffusion 共享规则](rules.md)。 +### DIFF-2ac — bare-transformer Cache-DiT adapter 与 request refresh 不得携带旧 engine 状态 + +- 触发：Cache-DiT `BlockAdapter` construction/enable、serial worker 中第二个 engine，或 runner cache refresh step resolution 变更。 +- 强制：bare-transformer adapter 每次 enable 前清除其 wrapper pipe class 的陈旧 `_is_cached` marker，再安装新的 cache context；该 marker 是 adapter wrapper class 的 process-wide 副作用，不是可跨 engine 复用的 installation state。每 request refresh 以显式 `num_inference_steps`、`len(timesteps)`、`len(sigmas)`、pipeline `default_num_inference_steps` 的顺序求步数；显式数值优先于 custom schedule 是当前合并源码的有意语义。 +- 禁止：因前一个 engine 未 disable 而跳过第二个 bare-transformer adapter 的 context installation；不得让 class marker 或 cache refresh 在 process/request 间 carry over，也不得把 schedule-first 的旧 review 建议当成已合并行为。 +- 验收：SANA CPU regression `test_second_engine_enables_cache_dit_after_undisabled_first` 覆盖 serial second-engine red→green；runner CPU test 覆盖 explicit/timesteps/sigmas/default 四段 refresh chain。现有证据不证明所有 declarative adapter 或跨进程 lifecycle 的同等行为。^[PR #5882] + ### DIFF-2f — 多 DiT pipeline 的 component list 是所有共享 lifecycle 的唯一集合 - 触发：pipeline 增加第二个 DiT、task-dependent transformer，或修改 `_dit_modules`。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 59b1fac49..598a6c1ec 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 95a93116` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3b6ce984` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e30b923c5..478738673 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 95a93116` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3b6ce984` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 029784fbe..0a11727a0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 95a93116` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3b6ce984` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c79f7afb7..429ef978a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 95a93116` 验证存在 +- 源码校验：以上路径均已在 `main @ 3b6ce984` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index bd79b4f32..b7056af60 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 95a93116`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3b6ce984`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from d5fd9eb6 \ - --to 95a93116 \ + --from 95a93116 \ + --to 3b6ce984 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/sana-video/_index.md b/knowledge/repos/vllm-omni/models/sana-video/_index.md index b2d758bf6..0b43d9d9d 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/_index.md +++ b/knowledge/repos/vllm-omni/models/sana-video/_index.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #5508", "PR #5861", "PR #6953", vllm_omni/diffusion/models/sana_video/, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/model_extras/sana_video.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] +sources: ["PR #5508", "PR #5861", "PR #5882", "PR #6953", vllm_omni/diffusion/models/sana_video/, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/registry.py, vllm_omni/model_extras/sana_video.py, tests/diffusion/models/sana_video/test_cache_offload.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] confidence: high --- @@ -76,4 +76,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| | T2V/I2V、checkpoint、VAE、adapter 或验证边界 | [architecture](architecture.md) | -| native TP/CFG topology、GLUMB/RMSNorm layout 或 I2V parallel invariant | [SANA rules](rules.md) | +| native TP/CFG topology、GLUMB/RMSNorm layout、I2V parallel invariant、Cache-DiT/offload admission 或 SANA 50-step refresh | [SANA rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/sana-video/architecture.md b/knowledge/repos/vllm-omni/models/sana-video/architecture.md index 69f1c4375..3268f205c 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/architecture.md +++ b/knowledge/repos/vllm-omni/models/sana-video/architecture.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: architecture tags: [vllm-omni, models, diffusion] -sources: ["PR #5508", "PR #5861", "PR #6953", vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, vllm_omni/diffusion/request.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] +sources: ["PR #5508", "PR #5861", "PR #5882", "PR #6953", vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, vllm_omni/diffusion/request.py, tests/diffusion/models/sana_video/test_cache_offload.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] confidence: high --- @@ -32,8 +32,12 @@ Diffusers-adapter 并非 native pipeline 的替代实现：它由 model class 特别是 checkpoint 声明的 T2V Diffusers class 在请求 I2V 时必须改装为 Diffusers I2V class，且 shared `num_frames` 要翻译为 Diffusers 的 `frames`。native pipeline 的 TP/CFG 各仅支持 `{1,2}`， 包括 TP2×CFG2；SP、text-encoder TP、PP、HSDP 在加载前拒绝，精确 layout/CFG/I2V invariant 见 -[SANA-1a](rules.md#sana-1a-native-sana-video-只支持-tpcfg-的-12-矩阵)。Cache-DiT、TeaCache 和 -step execution 仍未验证，adapter 也不提供 native continuous batching 或并行能力。 +[SANA-1a](rules.md#sana-1a-native-sana-video-只支持-tpcfg-的-12-矩阵)。native T2V/I2V 的 +Cache-DiT 与 CPU/layerwise offload 另限 TP=CFG=SP=1，且只允许 `cache_dit`（与 distributed +layerwise offload 不可组合）；Cache-DiT、TeaCache 和 step execution 不应混为同一能力，后两者仍 +未验证。加载在 default-device context 完成，offloader 拥有 runtime residency；精确 admission、 +hook lifecycle 与 refresh 规则见 [SANA-1b/1c](rules.md)。adapter 也不提供 native continuous +batching 或并行能力。 ## 从输入到输出的主要流程 diff --git a/knowledge/repos/vllm-omni/models/sana-video/rules.md b/knowledge/repos/vllm-omni/models/sana-video/rules.md index de6197765..9380e30c3 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/rules.md +++ b/knowledge/repos/vllm-omni/models/sana-video/rules.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5861", vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, tests/diffusion/models/sana_video/] +sources: ["PR #5861", "PR #5882", vllm_omni/diffusion/models/sana_video/pipeline_sana_video.py, vllm_omni/diffusion/models/sana_video/pipeline_sana_video_i2v.py, vllm_omni/diffusion/models/sana_video/transformer_sana_video.py, tests/diffusion/models/sana_video/, tests/diffusion/models/sana_video/test_cache_offload.py, tests/diffusion/test_diffusion_model_runner.py] confidence: high --- @@ -14,5 +14,19 @@ confidence: high - 触发：native SANA Video TP/CFG、transformer layout、I2V 或 Diffusers adapter topology 变更。 - 强制：TP/CFG 各只 `{1,2}`，TP2×CFG2 valid；SP/text-encoder TP/PP/HSDP load 前 reject，Diffusers adapter reject native TP/CFG。TP 用 QKV column/output row、distributed RMSNorm global sumsq/local weights，GLUMB packed `[value;gate]` 每 segment slice 后 local chunk2/reduction。CFG1 batch neg/pos 后 slice；CFG2 mixin branch gather/combine FP32。uninitialized group serial，initialized broken group errors propagate；I2V first frame/mask 不变。 -- 禁止：TP>2/CFG>2、cache/offload/step/support/perf claim。 +- 禁止：TP>2/CFG>2；不得把本规则的 native TP/CFG `{1,2}` 矩阵外推为 cache/offload 的支持范围。 - 验收：CPU topology/layout 与 tiny multiprocess TP/CFG matrix；A800 数字仅该环境观察。^[PR #5861] + +### SANA-1b — native SANA 的 Cache-DiT 与 offload 只在单 rank 路径组合 + +- 触发：native T2V/I2V 的 cache backend、CPU/layerwise offload、Transformer block metadata 或加载设备变更。 +- 强制：两种 native pipeline 只接受 `none` 或 `cache_dit` backend；Cache-DiT、model CPU offload、layerwise CPU offload（含 distributed layerwise）任一启用时均要求 TP=CFG=SP=1，并在加载任何 checkpoint component 前 reject。`cache_dit` 与 distributed layerwise offload 必须 reject：per-rank cache skip 会使 weight AllGather 失步。Transformer 以 `transformer_blocks` 的 ordinary layerwise block list 声明 Cache-DiT `ForwardPattern.Pattern_3`；loader 按 `torch.get_default_device()` 放置 component，runtime residency 由 offloader 而非 pipeline 强制 runtime device。若 cache skip 使正常 prefetch 未发生，下一 block 的 hook 自行 materialize；disable/异常 cleanup 必须移除 hooks 与 cache/offload state。 +- 禁止：TeaCache 或其他 backend、Cache-DiT + distributed layerwise offload、以及任何 TP/CFG/SP>1 的 cache/offload 组合；不得把单 rank distributed-layerwise admission 当作多 rank 验证，也不得称 native T2V/I2V 的 TeaCache 或 step execution 已支持/验证。 +- 验收：`test_cache_offload.py` CPU/mock 覆盖 adapter pattern、backend/parallel admission 在 component load 前失败、loader device、cache-skip 后 hook materialization 与 disable cleanup；tiny common harness 覆盖 Cache-DiT、两种 offload 及两个组合。作者报告单张 A800 的真实权重 T2V/I2V 480p/720p 19/19 matrix（distributed layerwise 仅单 rank）和 Cache-DiT/offload 指标；这些数值与 CPU/mock 不能外推到多 rank 或其他硬件。^[PR #5882] + +### SANA-1c — Cache-DiT refresh 使用 request 的有效 step 数 + +- 触发：native SANA sampling defaults、custom `timesteps`/`sigmas`，或 shared cache refresh 修改。 +- 强制：每 request refresh 的优先级固定为显式 `num_inference_steps`、`len(timesteps)`、`len(sigmas)`、pipeline `default_num_inference_steps`；SANA T2V 与 I2V 的最后 fallback 都是 50。合并后的 shared source 有意保留显式 number 优先于 custom schedule 的顺序，不能把较早 review 讨论中的 schedule-first 建议误记为最终合同。 +- 禁止：在 request omit steps 时沿用上一个 request 的 cache state；不得把 50 当作 custom schedule 的长度，或声称 TeaCache/step execution 通过此 fallback 获得支持。 +- 验收：`tests/diffusion/test_diffusion_model_runner.py` 覆盖 explicit、timesteps、sigmas 和 default 的 refresh chain；`test_cache_offload.py` 覆盖 native 50-step contract。^[PR #5882] From c62406393ca3fd7459451045d99a5f8c67ec5f6e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:21:58 +0800 Subject: [PATCH 432/465] knowledge(vllm-omni): sync 3ab582c9 (PR #6113) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/qwen3-tts/_index.md | 8 ++++-- .../repos/vllm-omni/models/qwen3-tts/rules.md | 26 ++++++++++++++++--- 11 files changed, 41 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 18682c13b..6eece0caf 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 95a93116960e364cc759c49023a4c35284157235 + previous_audited_sha: 3b6ce98421a4a32ad2c0a44e241a43f29f2ce5fa audited_ref: main - audited_sha: 3b6ce98421a4a32ad2c0a44e241a43f29f2ce5fa + audited_sha: 3ab582c99352113000d61e4522c18324a301793a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 548313c50..50e99c88c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3b6ce98421a4a32ad2c0a44e241a43f29f2ce5fa`. The canonical machine baseline +3ab582c99352113000d61e4522c18324a301793a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 554f2d634..67d5c6c7b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3b6ce984` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 3ab582c9` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 79f4a5186..ba3101a6f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3b6ce984` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 3ab582c9` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 598a6c1ec..4830390ef 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3b6ce984` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 3ab582c9` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 478738673..1ba0f04c0 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3b6ce984` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 3ab582c9` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0a11727a0..0a38490cc 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3b6ce984` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 3ab582c9` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 429ef978a..8e0f91ca8 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3b6ce984` 验证存在 +- 源码校验：以上路径均已在 `main @ 3ab582c9` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index b7056af60..34d08002f 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3b6ce984`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 3ab582c9`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 95a93116 \ - --to 3b6ce984 \ + --from 3b6ce984 \ + --to 3ab582c9 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md index 9e8b25e79..18afc9c5a 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/_index.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: index tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6553", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py] +sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6113", "PR #6553", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py, vllm_omni/platforms/npu/layers/rotary_embedding.py, tests/entrypoints/openai_api/test_serving_speech.py] confidence: high --- @@ -29,5 +29,9 @@ confidence: high - 审查 `x_vector_only_mode`、ICL、`ref_audio` artifact-only reuse、incremental Code2Wav、segmented CUDA Graph、engine 存活性，或 NPU tokenizer RoPE 的 BNSD/BSND shape fallback。 -- 描述直达源码与具体不变量见 [Qwen3-TTS rules](rules.md#direct-代码快速入口)；模型家族结构见 +- 审查 Qwen3-TTS 的 effective `task_type`、uploaded/precomputed stored voice、 + `tts_model_type`/model-path checkpoint variant 与 Base speaker-embedding mismatch + admission 时，直接看 [Q3TTS-1d](rules.md#q3tts-1d-effective-task-必须在-dispatch-前与-checkpoint-variant-对齐)； + built-in `supported_speakers` 不属于 stored voice。 +- 其他源码入口与具体不变量见 Qwen3-TTS rules；模型家族结构见 [Qwen-Omni](../qwen-omni/_index.md)。 diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index f4d9a0d92..64e4db907 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, models, serving, qwen-omni] -sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6523", "PR #6728", "PR #6861", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_speech_stream.py, vllm_omni/entrypoints/openai/speech_usage.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_serving_speech_stream.py, tests/entrypoints/openai_api/test_tts_adapter.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] +sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6113", "PR #6523", "PR #6728", "PR #6861", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_speech_stream.py, vllm_omni/entrypoints/openai/speech_usage.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_serving_speech_stream.py, tests/entrypoints/openai_api/test_tts_adapter.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] confidence: high --- @@ -17,9 +17,9 @@ confidence: high | PR 描述信号 | 规则组 | 第一批源码 | |---|---|---| | Qwen3-TTS、`qwen3_tts` pipeline | Q3TTS-1a/1b | `config/pipeline_registry.py::OMNI_PIPELINES["qwen3_tts"]`；`model_executor/models/qwen3_tts/pipeline.py` | -| `ref_audio`、x-vector、ICL、artifact-only reuse | Q3TTS-1a/1b/1c | `entrypoints/openai/serving_speech.py::_qwen3_tts_can_use_ref_audio_artifact_only`、`_track_ref_audio_artifact_warmup`、`_mark_ref_audio_artifact_ready_for_request` | +| `ref_audio`、stored voice、task/checkpoint variant、x-vector、ICL、artifact-only reuse | Q3TTS-1a/1b/1c/1d | `entrypoints/openai/tts_adapters/qwen3_tts.py::{validate,_get_model_variant}` → `entrypoints/openai/serving_speech.py::_build_tts_params`、`_qwen3_tts_can_use_ref_audio_artifact_only`、`_track_ref_audio_artifact_warmup`、`_mark_ref_audio_artifact_ready_for_request` | | talker/code2wav、adaptive chunk、delta frame、request cache、segmented graph | Q3TTS-3a/3b/3c/3d/3e + Model Executor | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `stage_input_processors/chunk_size_utils.py` → `qwen3_tts_code2wav.py::Qwen3TTSCode2Wav` → `segmented_graph_wrapper.py` | -| OpenAI speech adapter | Q3TTS-1a/1b + Serving | `entrypoints/openai/tts_adapters/qwen3_tts.py::Qwen3TTSAdapter` → `serving_speech.py` | +| OpenAI speech adapter | Q3TTS-1a/1b/1d + Serving | `entrypoints/openai/tts_adapters/qwen3_tts.py::Qwen3TTSAdapter` → `serving_speech.py` | | NPU、RoPE、BNSD/BSND、`codec_chunk_ramp` | Q3TTS-2a | `platforms/npu/models/qwen3_tts_tokenizer_v2.py::_apply_rotary_pos_emb_npu` → `platforms/npu/layers/rotary_embedding.py::npu_rotary_mul_with_bsnd_fallback` | ## Q3TTS-1a — ref-audio readiness 按 mode/capability 隔离 @@ -53,6 +53,26 @@ confidence: high - 验收：ready 的实际 marker selector 收集并执行该 case；x-vector → ICL → x-vector 在同一 audio 下通过，且在旧的 mode-agnostic artifact readiness 实现上失败。 ^[PR #5157] ^[PR #6523] +## Q3TTS-1d — effective task 必须在 dispatch 前与 checkpoint variant 对齐 + +- 触发：修改 Qwen3-TTS Speech API 的 `task_type` 推断、uploaded/precomputed voice、checkpoint + variant 识别，或 Base prompt 的 speaker embedding 拼接。 +- 强制：先确定 effective task：未显式 task 的 `ref_audio`/`ref_text` 选择 Base；uploaded voice 或 + `capabilities.precomputed_speakers` 中的 voice 无条件选择 Base，即使 caller 给了另一 task； + `capabilities.supported_speakers` 的 built-in preset 仍是 CustomVoice。再以 + `hf_config.tts_model_type` field-first 解析已识别 variant；field 缺失或未识别时，才从 model path + 的 leaf 向上逐 component 查找 token-delimited marker。识别到 variant 且与 effective task 不同， + 必须在 engine dispatch 前返回 public 400，明确 task、loaded variant 和可用 alternative；无法识别 + variant 返回 `None` 并 fail-open。 +- 禁止：在 validation 后才把 stored voice 改写为 Base；读取 `server.precomputed_speakers` 而不是 + adapter capability；把 `base_models`、`database` 或 parent substring 当 Base marker；让已知 + task/variant mismatch 进入 prompt builder。prompt builder 的 speaker-dimension guard 仅保留为绕过 + Speech API 时的 diagnostic，仍可能使 EngineCore 终止，不得把它宣传为 service-preserving admission。 +- 验收：focused tests 覆盖 uploaded 与 precomputed stored voice（包括 caller 的 conflicting task）、 + built-in speaker、metadata priority over conflicting/uninformative path、unknown/unrecognized metadata 的 + token-delimited leaf-up fallback 与 unknown fail-open，以及三种 known task/variant mismatch 在 engine + dispatch 前为 public 400；另验证 prompt-builder dimension mismatch diagnostic。 ^[PR #6113] + ## Q3TTS-2a — NPU RoPE 按真实 shape 在 BNSD 与 BSND fused path 间选择 - 触发：Qwen3-TTS 12 Hz tokenizer decoder 的 NPU RoPE，尤其启用短 From 0b533e7d31ebb6d0a178ccab7e89704822a06595 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:25:57 +0800 Subject: [PATCH 433/465] knowledge(vllm-omni): sync b9a88bf5 (PR #6957) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/minicpm-o-4-5/rules.md | 32 +++++++++---------- .../vllm-omni/models/step-audio2/_index.md | 4 +-- .../models/step-audio2/architecture.md | 13 +++++--- .../vllm-omni/models/step-audio2/rules.md | 24 ++++++++++++-- 13 files changed, 60 insertions(+), 37 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6eece0caf..372fcb0b8 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3b6ce98421a4a32ad2c0a44e241a43f29f2ce5fa + previous_audited_sha: 3ab582c99352113000d61e4522c18324a301793a audited_ref: main - audited_sha: 3ab582c99352113000d61e4522c18324a301793a + audited_sha: b9a88bf51e2e7a175a6b559db1d442560551c581 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 50e99c88c..0ec9ffa09 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -3ab582c99352113000d61e4522c18324a301793a`. The canonical machine baseline +b9a88bf51e2e7a175a6b559db1d442560551c581`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 67d5c6c7b..ef205a95b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 3ab582c9` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b9a88bf5` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index ba3101a6f..a69f7fa43 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 3ab582c9` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ b9a88bf5` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4830390ef..f153b8c5d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 3ab582c9` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b9a88bf5` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1ba0f04c0..6732c3d0e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 3ab582c9` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b9a88bf5` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 0a38490cc..4150a5991 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 3ab582c9` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b9a88bf5` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8e0f91ca8..c705d148f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 3ab582c9` 验证存在 +- 源码校验：以上路径均已在 `main @ b9a88bf5` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 34d08002f..0bd5009be 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 3ab582c9`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b9a88bf5`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 3b6ce984 \ - --to 3ab582c9 \ + --from 3ab582c9 \ + --to b9a88bf5 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 24a058f17..2da88ff7e 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -55,22 +55,22 @@ confidence: high - 触发：`token2wav_trt` / `MINICPMO_TOKEN2WAV_TRT=1`，或修改 Code2Wav 的 DiT、 Campplus、ONNX/TRT engine cache 与 optimization profile。 -- 强制：所有平台都先构造树内 `MiniCPMO45Token2wav`；开关只在 CUDA 同时启用 DiT - estimator 与 Campplus 的 TensorRT 路径，非 CUDA 忽略开关，关闭开关保留完整 torch - 路径。DiT 只替换 `BatchedToken2Wav._estimator_step` 的逐步 estimator 调用，encoder 与 - HiFT 仍走 torch；engine 在调用者当前 CUDA stream 上执行。 -- 强制：强类型精度来自所选 dtype 的 ONNX graph，export 深拷贝已加载 DiT 后再 cast， - 不改变 live torch estimator 的 dtype。ONNX cache 是动态、可移植 graph；plan cache key - 必须精确包含 depth、dtype、`token2wav_trt_max_batch`、GPU SM 与 TensorRT version。 -- 强制：ONNX/plan 首次生成使用 PID 唯一临时文件并原子替换；step 在执行前检查 profile： - batch 上限为配置值且包含 classifier-free guidance 展开的 `2B`，chunk 固定上限 3000， - attention cache 固定上限 1000，越界给出可操作错误。Campplus 合同保持 `[T,80]` → - `[1,192]` fp32，并从 `_resolve_model_dir()` 已解析的本地模型目录取资产。 -- 禁止：把共享 `step_audio2_dit_trt.py` 的存在解释成 Step-Audio2 用户自动获得 TRT； - 当前开关只由 MiniCPM-o wiring 调用。也禁止把讨论中的通用 vocoder backend 当成已实现。 -- 验收：分别覆盖关闭/开启、非 CUDA 忽略、torch/TRT 数值与音频 parity、首次构建和 cache - 复用、不同 SM 不共享 plan、CFG 后 batch 越界以及 chunk/cache 越界。PR 未新增自动化测试， - 因而这些仍是后续改动必须补证的验证缺口。 ^[PR #5638] +- 强制：各平台先构造树内 `MiniCPMO45Token2wav`；仅 CUDA 开关同时启用 DiT estimator 与 + Campplus TRT，关闭/非 CUDA 保留 torch。DiT 只替换 `_estimator_step`，encoder/HiFT 仍为 + torch 且 engine 用调用者 CUDA stream。dtype ONNX export 深拷贝再 cast、不改 live estimator； + plan key 含 depth、dtype、max-batch、SM、TRT version，profile 覆盖 CFG `2B`、chunk 3000、 + cache 1000，越界报可操作错误；Campplus 是本地资产 `[T,80]` → `[1,192]` fp32。 +- 强制：共享 `_publish_atomically` 用同目录 `.tmp..` 独占 `xb`；writer + 写入、flush/close 后 `os.replace`，并发成功完整且 last-writer-wins。collision 在 writer 前 + 失败且不删 foreign tmp；任何失败保留 destination、只删 owned tmp、重抛原异常，cleanup 仅 + 日志。ONNX path writer 在 umask 移除 owner-write 时记录 mode、补写权限并在发布前恢复；plan + stream 不作 `fstat`/`fchmod`。无 fsync/目录持久化、锁、内容校验或 ONNX/plan 成对事务。 +- 禁止：把 shared helper 解释成 Step-Audio2 TRT；当前显式 consumer 只有 MiniCPM-o Code2Wav， + 也不得把讨论中的通用 vocoder backend 当作已实现。 +- 验收：覆盖开关/非 CUDA、torch/TRT parity、cache/SM/profile 边界；运行 + `python -m pytest -q tests/model_executor/models/step_audio2/test_step_audio2_dit_trt.py` 覆盖 + CPU 并发、collision、writer/fstat/replace/cleanup failure 与 restrictive umask。MiniCPM + wiring 的自动化验证仍须补证。 ^[PR #5638] ^[PR #6957] ## MCPMO-1f — Ascend Code2Wav NPUGraph 只捕获精确形状的 CFM DiT diff --git a/knowledge/repos/vllm-omni/models/step-audio2/_index.md b/knowledge/repos/vllm-omni/models/step-audio2/_index.md index 032facab2..a2aedeba8 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/_index.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/_index.md @@ -4,7 +4,7 @@ created: 2026-07-21 updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: ["PR #5067", "PR #5638", "PR #5869", "PR #5917", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py, vllm_omni/platforms/npu/models/step_audio2_token2wav.py, vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py] +sources: ["PR #5067", "PR #5638", "PR #5869", "PR #5917", "PR #6957", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/deploy/step_audio_2.yaml, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py, tests/model_executor/models/step_audio2/test_step_audio2_dit_trt.py, vllm_omni/platforms/npu/models/step_audio2_token2wav.py, vllm_omni/platforms/npu/models/cosyvoice2_dit_attn.py] --- # Step-Audio2 @@ -50,7 +50,7 @@ sources: ["PR #5067", "PR #5638", "PR #5869", "PR #5917", vllm_omni/model_execut | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | token 过滤桥、25+3 lookahead、流式状态 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | -| 设备常驻 token、ragged 长度同步与性能证据边界 | [rules](rules.md) | Step-Audio2 专属审查规则 | +| 设备常驻 token、ragged 长度同步、TRT 临时文件发布 | [rules](rules.md) | Step-Audio2 专属审查规则 | ## 配置与 checkpoint 差异（三变体） diff --git a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md index 756f07ae9..b70b3851f 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/architecture.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-21 updated: 2026-09-05 type: architecture tags: [vllm-omni, models] -sources: ["PR #5067", "PR #5638", "PR #5869", "PR #5917", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py, tests/model_executor/models/step_audio2/test_step_audio2_token2wav_async_chunk.py] +sources: ["PR #5067", "PR #5638", "PR #5869", "PR #5917", "PR #6957", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, vllm_omni/model_executor/models/step_audio2/step_audio2_constants.py, vllm_omni/model_executor/stage_input_processors/step_audio2.py, tests/model_executor/models/step_audio2/test_hift_parity.py, tests/model_executor/models/step_audio2/test_step_audio2_token2wav_async_chunk.py, tests/model_executor/models/step_audio2/test_step_audio2_dit_trt.py] --- # Step-Audio2 架构 @@ -26,9 +26,11 @@ sources: ["PR #5067", "PR #5638", "PR #5869", "PR #5917", vllm_omni/model_execut s3tokenizer（prompt wav 语音 token）+ ONNX 说话人嵌入 + hyperpyyaml 加载的 flow-matching（10 步 ODE）+ 树内 CosyVoice3 HiFT/mel 实现 → 24 kHz;树内带说话人 wav——`assets/default_female.wav` 是默认,`default_male.wav` 是备选。 -- 共享加速实现：`step_audio2_dit_trt.py` 提供流式 DiT ONNX export/TRT stepper， - `step_audio2_token2wav.py` 提供 Campplus TRT helper；在此 pin 上只有 MiniCPM-o 的 - Code2Wav wiring 显式启用它们，Step-Audio2 pipeline 不会因共享代码存在而自动切换。 +- 共享加速实现：`step_audio2_dit_trt.py` 提供流式 DiT ONNX export/TRT stepper，并以同目录 + 独占临时文件发布 ONNX/plan（成功时原子 replace、并发 last-writer-wins）； + `step_audio2_token2wav.py` 提供 Campplus TRT helper。在此 pin 上只有 MiniCPM-o 的 Code2Wav + wiring 显式启用它们，Step-Audio2 pipeline 不会因共享代码存在而自动切换；发布器不提供锁、 + fsync/目录持久化、内容校验或 ONNX/plan 成对事务。^[PR #6957] - 常量单一来源 `step_audio2_constants.py`：文本 ≤151688;音频 token **151696–158257**（`audio_vocab_size` 6562,相对 `audio_eos` 6561）;流式 `chunk_size 25` / `pre_lookahead_len 3` / mel cache 8 帧。 @@ -76,7 +78,8 @@ pin 上只有**功能面**验证入口;无精度基线或性能 gate 证据,下 精度/性能维度,相关结论需另行实测。 - 单元：`tests/model_executor/models/step_audio2/`（thinker、token2wav - async chunk）、`tests/model_executor/stage_input_processors/test_step_audio2_async_chunk.py`; + async chunk、`test_step_audio2_dit_trt.py` 的 CPU 原子发布）、 + `tests/model_executor/stage_input_processors/test_step_audio2_async_chunk.py`; NPU:`tests/platforms/npu/test_step_audio2_token2wav.py`;parser: `tests/reasoning/test_step_audio_reasoning_parser.py`。 - e2e：`tests/e2e/{offline_inference,online_serving}/test_step_audio2_expansion.py` diff --git a/knowledge/repos/vllm-omni/models/step-audio2/rules.md b/knowledge/repos/vllm-omni/models/step-audio2/rules.md index a85ff2a09..2cc9672b3 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/rules.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/rules.md @@ -1,10 +1,10 @@ --- title: "Step-Audio2 设备边界与流式性能规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models] -sources: ["PR #5067", vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, tests/model_executor/models/step_audio2/test_step_audio2_token2wav_async_chunk.py, tests/platforms/npu/test_step_audio2_token2wav.py, "PR #6467"] +sources: ["PR #5067", vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py, vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py, vllm_omni/model_executor/models/step_audio2/step_audio2_dit_trt.py, tests/model_executor/models/step_audio2/test_step_audio2_token2wav_async_chunk.py, tests/model_executor/models/step_audio2/test_step_audio2_dit_trt.py, tests/platforms/npu/test_step_audio2_token2wav.py, "PR #6467", "PR #6957"] --- # Step-Audio2 设备边界与流式性能规则 @@ -62,3 +62,23 @@ sources: ["PR #5067", vllm_omni/model_executor/models/step_audio2/step_audio2_th - 强制：音频读取统一调用 `load_audio(..., sr=None, mono=True)`，mel filter 统一调用 `mel_filter_bank`；16 kHz 的 s3tokenizer/说话人分支和 24 kHz flow-mel 分支都必须从原始 waveform 分别经 `AudioResampler` 生成，保留真实 sample rate 与单声道语义。 - 禁止：在目标运行时重新导入 `librosa` 或 `soundfile`，通过 try/except 在两套解码实现间兜底，或把已经 16 kHz 的结果再次作为 24 kHz 重采样输入；不得据此声称 onnxruntime/s3tokenizer 等其他依赖已移除。 - 验收：静态检查 Step-Audio2 runtime 不再依赖 `librosa`/`soundfile`；覆盖非 16/24 kHz 输入，断言两个目标采样率都由同一原始音频独立生成、mel shape/数值和 token2wav 推理结果正常，并运行 `pytest tests/e2e/online_serving/test_step_audio2_expansion.py --run-level 'advanced_model'`。 ^[PR #6467] + +## STEPA2-2b — DiT TRT 构建必须独占临时文件后原子发布 + +- 触发：修改 `step_audio2_dit_trt.py` 的 ONNX export、TRT plan 序列化、缓存发布或共享 + `Code2Wav` 的 DiT/Campplus TRT wiring。 +- 强制：共享 `_publish_atomically` 在 destination 同目录以 + `.tmp..` 独占 `xb` 创建临时文件；writer 写完，stream flush/close 后才 + `os.replace`。并发成功者均发布完整 artifact，最终允许 last-writer-wins。名字碰撞须在 + writer 前抛出且不得删除 foreign tmp；writer、权限处理或 replace 任一失败必须保留已有 + destination、只删除本调用拥有的 tmp、重抛原异常；cleanup 异常只写日志。ONNX path writer + 在 umask 移除 owner write 时必须记录 mode、临时补 owner-write，并在发布前恢复；plan 的 + stream writer 不做 `fstat`/`fchmod`。 +- 边界：这不是 fsync 或目录持久化合同，不提供锁、内容校验或 ONNX/plan 成对事务。共享 helper + 位于 Step-Audio2 并不代表其 pipeline 已启用 TRT：此 pin 的显式 consumer 是 MiniCPM-o + Code2Wav；不得把该 wiring 或 TRT 支持归因给 Step-Audio2 用户。 +- 验收：运行 `python -m pytest -q tests/model_executor/models/step_audio2/test_step_audio2_dit_trt.py`； + CPU 测试必须覆盖同进程并发、foreign-name collision、writer/fstat/replace failure、cleanup + failure 与 restrictive umask 的 ONNX mode 恢复，并断言 plan 无 `fstat`/`fchmod`。更广的 + Step-Audio2 测试收集在该 PR 环境因 optional `s3tokenizer` 缺失而无法完成，不能误报为完整 + suite 已通过。 ^[PR #6957] From 563a126aa9b13595491bc37795c0247bf38b29bf Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:32:25 +0800 Subject: [PATCH 434/465] knowledge(vllm-omni): sync ca6b4e21 (PR #6317) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../model-executor/rules-runtime-hot-paths.md | 10 +++--- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../vllm-omni/models/omnivoice/_index.md | 18 ++++++++--- .../repos/vllm-omni/models/omnivoice/rules.md | 32 +++++++++++++++++++ 12 files changed, 62 insertions(+), 22 deletions(-) create mode 100644 knowledge/repos/vllm-omni/models/omnivoice/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 372fcb0b8..e7e794c98 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 3ab582c99352113000d61e4522c18324a301793a + previous_audited_sha: b9a88bf51e2e7a175a6b559db1d442560551c581 audited_ref: main - audited_sha: b9a88bf51e2e7a175a6b559db1d442560551c581 + audited_sha: ca6b4e21127ff3f36530438a42948fe17117d412 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0ec9ffa09..11ac2c479 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b9a88bf51e2e7a175a6b559db1d442560551c581`. The canonical machine baseline +ca6b4e21127ff3f36530438a42948fe17117d412`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ef205a95b..46bc9c6ab 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b9a88bf5` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ca6b4e21` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a69f7fa43..10745e9e7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b9a88bf5` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ ca6b4e21` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index f153b8c5d..7e2c70eac 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b9a88bf5` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ca6b4e21` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6732c3d0e..f2ddf3031 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b9a88bf5` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ca6b4e21` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md index 8e3b1c196..986fc4cf2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-runtime-hot-paths.md @@ -1,10 +1,10 @@ --- title: "运行时热路径合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048", "PR #6424", "PR #6454", vllm_omni/data_entry_keys.py, "vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py", "vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py", "vllm_omni/model_executor/stage_input_processors/cosyvoice3.py", "PR #6458"] +sources: ["PR #4765", "PR #5068", "PR #5174", "PR #5666", vllm_omni/worker/, "PR #5452", "vllm_omni/worker/sparse_audio.py", "vllm_omni/worker/sampling_utils.py", "PR #5048", "PR #6424", "PR #6454", vllm_omni/data_entry_keys.py, "vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py", "vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py", "vllm_omni/model_executor/stage_input_processors/cosyvoice3.py", "PR #6458", "PR #6317"] confidence: high --- @@ -22,9 +22,9 @@ confidence: high ## EXEC-9a — OmniVoice 热路径的固定输入缓存与掩码/精度边界 - 触发：修改 OmniVoice 离散扩散生成循环中的 D2H 同步、attention mask、RoPE、文本/音频 embedding、CFG、CUDA graph/eager 路径或 TF32 配置。 -- 强制：将循环不变的文本 embedding、`audio_mask_3d`、RoPE 和 additive attention mask 在循环外缓存；批量取得 `c_lens` 并只在实际消费的 logits slice 上转为 fp32；CUDA graph 与 eager 使用相同的 float mask 语义，bool mask 必须将 True 映射为 `0.0`、False 映射为 `-inf`，已有 float mask 必须原样保留；TF32 只能通过显式配置 opt-in，并在 graph capture 前启用。 -- 禁止：每步重复计算固定输入或执行逐请求 D2H `.item()`；对 float mask 使用布尔反转或把 bool 直接隐式转换进 float buffer；让 graph capture 与 replay 使用不同 mask dtype；默认开启或宣称 TF32 路径 bit-identical；只更新 conditional 或 unconditional 一侧的迭代 token。 -- 验收：覆盖 eager 与 CUDA graph、bool 与 additive float mask、CFG/non-CFG 及不同 batch/concurrency，断言 masked 位置仍为 `-inf`、float mask 数值保留、graph/eager 输出满足既定精度合同、每步无固定输入重算和逐请求同步；单独验证 `enable_tf32` 默认关闭且显式开启前后性能/非 bit-identical 语义有记录。 ^[PR #5174] +- 强制：将循环不变的文本 embedding、`audio_mask_3d`、RoPE 和 additive attention mask 在循环外缓存；批量取得 `c_lens` 并只在实际消费的 logits slice 上转为 fp32；普通缓存的 additive mask、CUDA graph capture buffer 与 replay 前 normalization 都使用 `OmniVoiceGenerator.model_dtype`。CUDA graph 与 eager 使用相同的 float mask 语义：bool mask 必须将 True 映射为 `0.0`、False 映射为 `-inf`，已有 float mask 必须原样保留；TF32 只能通过显式配置 opt-in，并在 graph capture 前启用。 +- 禁止：每步重复计算固定输入或执行逐请求 D2H `.item()`；为 additive mask 恢复 fp32 默认值、对 float mask 使用布尔反转，或把 bool 直接隐式转换进 float buffer；让 graph capture 与 replay 使用不同 mask dtype；默认开启或宣称 TF32 路径 bit-identical；只更新 conditional 或 unconditional 一侧的迭代 token。 +- 验收：覆盖 eager 与 CUDA graph、bool 与 additive float mask、CFG/non-CFG 及不同 batch/concurrency，断言 masked 位置仍为 `-inf`、float mask 数值保留，三处 mask dtype 都等于 `model_dtype`，graph/eager 输出满足既定精度合同、每步无固定输入重算和逐请求同步；单独验证 `enable_tf32` 默认关闭且显式开启前后性能/非 bit-identical 语义有记录。 ^[PR #5174] ^[PR #6317] ## EXEC-11a — 连续 AR 音频侧路必须保持 fp32、上下文与正常终止 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 4150a5991..8ebd56764 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b9a88bf5` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ca6b4e21` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c705d148f..48bf7eb75 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b9a88bf5` 验证存在 +- 源码校验：以上路径均已在 `main @ ca6b4e21` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 0bd5009be..51c6abc1e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b9a88bf5`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ca6b4e21`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 3ab582c9 \ - --to b9a88bf5 \ + --from b9a88bf5 \ + --to ca6b4e21 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/omnivoice/_index.md b/knowledge/repos/vllm-omni/models/omnivoice/_index.md index dc10bfc99..a98572813 100644 --- a/knowledge/repos/vllm-omni/models/omnivoice/_index.md +++ b/knowledge/repos/vllm-omni/models/omnivoice/_index.md @@ -1,15 +1,16 @@ --- title: "OmniVoice（离散扩散 TTS,AR/diffusion 双注册）" created: 2026-07-21 -updated: 2026-07-21 +updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/model_executor/models/omnivoice/, vllm_omni/diffusion/models/omnivoice/, vllm_omni/deploy/omnivoice.yaml] +sources: ["PR #6317", vllm_omni/model_executor/models/omnivoice/, vllm_omni/model_executor/models/omnivoice/omnivoice_generator.py, vllm_omni/model_executor/models/omnivoice/fused_qkv_rope.py, vllm_omni/diffusion/models/omnivoice/, vllm_omni/deploy/omnivoice.yaml] --- # OmniVoice -以下事实在 `main @ 5d44868e` 复核（源码派生页,尚无本模型的运行经验沉淀）。 +以下结构事实在 `main @ ca6b4e21` 复核；模型热路径的可执行门禁见 +[OmniVoice rules](rules.md#direct-代码快速入口)。 ## 名称与范围 @@ -31,10 +32,14 @@ sources: [vllm_omni/model_executor/models/omnivoice/, vllm_omni/diffusion/models `OmniVoiceDecoder`/`RuleDurationEstimator`,一次 `forward()` 完成 文本→unmask→8-codebook token→DAC 解码;可选 `transformers.HiggsAudioV2TokenizerModel` import 有 guard。 -- `omnivoice_generator.py`（39 KB）带 `_OmniVoiceCUDAGraphForward`;无 - stage_input_processor（单 stage 无交接）。 +- `omnivoice_generator.py` 带 `_OmniVoiceCUDAGraphForward`;无 + stage_input_processor（单 stage 无交接）。packed QKV、Q/K RMSNorm + RoPE、GQA + broadcast、SwiGLU packed activation、residual threading 及其 eager fallback 的边界见 + `OMNIVOICE-1a`。 - deploy `omnivoice.yaml` 未 pin checkpoint;`enforce_eager`、 `dtype: float32`、`distributed_executor_backend: mp`。 +- 该部署默认值仍为 float32；PR #6317 的 CUDA 测试仅狭义覆盖 fp16/bf16 热路径， + 不改变默认值，也不构成其他硬件、shape 或 serving 的支持声明。 - **未决**：AR-registry 入口 `OmniVoiceModel` 何时被走到无法仅从源码判定 （服务路径是单 DIFFUSION stage,docstring 暗示两 stage LLM_AR 形态）—— 断言其是否使用前需 live 验证。另注:`OmniVoice` 只是架构别名,不是独立 @@ -43,4 +48,7 @@ sources: [vllm_omni/model_executor/models/omnivoice/, vllm_omni/diffusion/models ## 什么时候查这里 - 审查 omnivoice 的双注册、unmask 步数/codebook 或 speaker cache 改动。 +- 审查 fused projection loading、Q/K/V + gate/up packing、QK norm/RoPE、GQA、SwiGLU、 + residual、mask dtype、CUDA graph 或 Triton fallback 时先看 `OMNIVOICE-1a`；共享固定输入/TF32 约束见 + [EXEC-9a](../../components/model-executor/rules-runtime-hot-paths.md#exec-9a-omnivoice-热路径的固定输入缓存与掩码精度边界)。 - 共享 RNG/graph 规则见 [Diffusion rules](../../components/diffusion/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/omnivoice/rules.md b/knowledge/repos/vllm-omni/models/omnivoice/rules.md new file mode 100644 index 000000000..f2744f2bf --- /dev/null +++ b/knowledge/repos/vllm-omni/models/omnivoice/rules.md @@ -0,0 +1,32 @@ +--- +title: "OmniVoice 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, model-executor, attention-runtime, ffn-runtime] +sources: ["PR #6317", vllm_omni/model_executor/models/omnivoice/omnivoice_generator.py, vllm_omni/model_executor/models/omnivoice/fused_qkv_rope.py, tests/model_executor/models/omnivoice/test_cuda_graph_generator.py, tests/model_executor/models/omnivoice/test_fused_projection_load.py, tests/model_executor/models/omnivoice/test_fused_qkv_rope.py, tests/model_executor/models/omnivoice/test_mask_dtype.py, tests/model_executor/models/omnivoice/test_triton_kernels.py] +confidence: high +--- + +# OmniVoice 规则 + +只有 `OMNIVOICE-数字字母` 是可审计规则 ID。双注册、pipeline 与部署结构见 +[_index](./_index.md)；共享固定输入、mask 与 TF32 约束见 +[EXEC-9a](../../components/model-executor/rules-runtime-hot-paths.md#exec-9a-omnivoice-热路径的固定输入缓存与掩码精度边界)。 + +## Direct 代码快速入口 + +| 正在做什么 | 精确规则组 | 第一批 live 源码 | +|---|---|---| +| OmniVoice generator 的 fused projection、checkpoint loading、attention prologue、SwiGLU、residual、mask dtype、CUDA graph 或 Triton fallback | `core`：`OMNIVOICE-1a` | `vllm_omni/model_executor/models/omnivoice/omnivoice_generator.py::{OmniVoiceGenerator._load_fused_projections,OmniVoiceAttention.forward,OmniVoiceMLP.forward,OmniVoiceTransformerBlock.forward,_OmniVoiceCUDAGraphForward}` → `vllm_omni/model_executor/models/omnivoice/fused_qkv_rope.py::{fused_qkv_norm_rope,fused_cuda_supported,_eager_qkv_norm_rope}` | + +| 审查组 | 什么时候触发 | 规则 ID | +|---|---|---| +| `core` | 每次命中 OmniVoice generator 热路径或其 checkpoint/attention 测试 | `OMNIVOICE-1a` | + +## OMNIVOICE-1a — OmniVoice 生成器热路径必须保持 pack、加载、注意力与回退合同 + +- 触发：修改 OmniVoice 的 Q/K/V 或 gate/up projection、checkpoint loader、Q/K RMSNorm/RoPE、GQA、SwiGLU、residual threading、attention mask、CUDA graph，或 `fused_qkv_rope.py` Triton/eager 路径。 +- 强制：checkpoint pack 顺序固定为 `qkv=[q,k,v]`、`gate_up=[gate,up]`；一旦某层开始 pack，缺少或仅出现部分 source shards 必须拒绝，不能让 fused parameter 保持随机初始化；未进入任何 fused-source group 的 shard 保持 generic loader 处理。attention prologue 只以 fp32 进行 Q/K RMSNorm、Q/K 权重计算与 Q/K RoPE，V 保持原 projection 值；随后按 GQA repeat K/V，并产出 contiguous `[batch, heads, sequence, head_dim]`（BHSD）。Triton 仅在 Triton 可用、CUDA tensor、QKV contiguous、head dim 为正的 2 的幂、Q 与 KV head 数都可被 8 整除且 GQA 可整除时启用；其他情况走 eager reference。residual 的加法/归一化代数不变，SwiGLU 保持 `silu(gate) * up` 的顺序。 +- 禁止：改变 source shard packing 次序、接受 partial/missing pack group，或把没有 fused source 的整组 shards 误判为 partial；对 V 应用 Q/K norm 或 RoPE；重排 GQA 的 K/V 映射、返回非 contiguous BHSD；把 Triton 的存在或 CUDA 单项条件当作充分资格，或删除 eager fallback；为方便而给 generator mask 重新引入 fp32 默认。不得把本 PR 的 A800、CPU 或 CUDA 测试/性能证据外推为其他硬件、shape、后端或普遍 serving support。 +- 验收：CPU 覆盖 pack order、全缺 source group 不触发 pack、任一 partial/missing 或错误 shape 拒绝，以及 eager 输出；CUDA+Triton 仅在上述 geometry 下对 eager reference 覆盖 fp32/fp16/bf16、Q/K norm+RoPE、未经变换的 V、正确 GQA index 与 contiguous BHSD；无 Triton、CPU、non-contiguous 或不合资格 geometry 必须回退。mask 测试覆盖 `model_dtype` 的 bool True→`0.0`、False→`-inf`、float 原样保留和 graph replay；graph/eager 及 residual/SwiGLU 回归保持既有精度合同。A800 上的 float16/float32 性能和 CPU/CUDA 单测只构成该 PR 的有界证据，不构成广泛性能或支持承诺。 ^[PR #6317] From faf0d05d95785e20b4fe49f5e887321c07050425 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:35:28 +0800 Subject: [PATCH 435/465] knowledge(vllm-omni): sync 63ed0fde (PR #6707) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-app-assembly.md | 22 +++++++++++------- .../serving/rules-upstream-compat.md | 23 +++++++++++-------- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 11 files changed, 39 insertions(+), 30 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e7e794c98..b7ff9449b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b9a88bf51e2e7a175a6b559db1d442560551c581 + previous_audited_sha: ca6b4e21127ff3f36530438a42948fe17117d412 audited_ref: main - audited_sha: ca6b4e21127ff3f36530438a42948fe17117d412 + audited_sha: 63ed0fdef4e6e9bdd866ea418ad9cf65e498c67a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 11ac2c479..4d9c74372 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ca6b4e21127ff3f36530438a42948fe17117d412`. The canonical machine baseline +63ed0fdef4e6e9bdd866ea418ad9cf65e498c67a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 46bc9c6ab..740e63f91 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ca6b4e21` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 63ed0fde` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 10745e9e7..aa75d50b9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ca6b4e21` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 63ed0fde` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 7e2c70eac..4d60ca60f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ca6b4e21` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 63ed0fde` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index f2ddf3031..e9b7bd728 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ca6b4e21` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 63ed0fde` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8ebd56764..f042c0aea 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ca6b4e21` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 63ed0fde` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 48bf7eb75..e90af203d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ca6b4e21` 验证存在 +- 源码校验：以上路径均已在 `main @ 63ed0fde` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md b/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md index 084985c17..fd88f8d5d 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md @@ -1,10 +1,10 @@ --- title: "Serving app assembly 规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #6609", vllm_omni/entrypoints/openai/api_server.py, tests/entrypoints/openai/test_profiler_endpoints.py, tests/entrypoints/openai_api/test_api_server_guards.py] +sources: ["PR #6609", "PR #6707", vllm_omni/config/endpoint_policy.py, vllm_omni/entrypoints/openai/api_server.py, tests/config/test_endpoint_policy.py, tests/entrypoints/openai/test_profiler_endpoints.py, tests/entrypoints/openai_api/test_api_server_guards.py] confidence: high --- @@ -14,14 +14,20 @@ confidence: high - 触发：assembled app 用 Omni handler 覆盖 upstream HTTP endpoint，尤其是附带可选 request body 或 stage scope 的 conditional router。 -- 强制：在 `include_router()` 前，从 assembled app 移除相同 path 与 HTTP method 的 upstream route； - 只在该 feature 启用的同一条件分支执行替换。FastAPI/Starlette 不会因后注册同 path/method 而替换 - 旧 route，dispatch 会命中先注册者。保留无关 upstream routes。 +- 强制：唯一公开 helper `endpoint_policy.remove_route_from_app(app, path, methods)` 的 `methods` 为必传 + 的 method set、没有 `None`/wildcard 语义；它只移除 `Route`、path 精确相等、`route.methods` 非 `None` + 且与所给 set 相交的条目，保留非 `Route`、不同 path 与 method 不相交的 routes。API server 必须导入 + 此 helper，不得保留第二份实现或由 endpoint policy 反向导入 API server。调用在 `include_router()` 前， + 并置于 feature 启用的同一条件分支；用于 endpoint restrictions 与 API/profiler overrides。FastAPI/ + Starlette 不会因后注册同 path/method 而替换旧 route，dispatch 会命中先注册者。 - 禁止：只检查 router 自身存在，或只断言 HTTP 200；这不能证明 assembled app 没有重复 route， 也不能证明 request body 到达 Omni handler。 -- 验收：assembly test 为 upstream 和 Omni 注册相同 method/path，断言仅剩一个 route 且 endpoint - 属于 Omni；公开 endpoint test 发送 `stages` 并断言 engine client 收到原值。当前 - `/start_profile` 与 `/stop_profile` 是此合同的实例。^[PR #6609] +- 验收：unit test 构造同 path 的 `Route`（相交和不相交 methods）及非 `Route`，断言仅移除相交者； + assembly test 验证 API server 使用同一 helper、没有本地 duplicate/circular import，并为 upstream 和 + Omni 注册相同 method/path 后断言仅剩 Omni endpoint。公开 endpoint test 发送 `stages` 并断言 engine + client 收到原值。`/start_profile` 与 `/stop_profile` 是当前实例。PR #6707 现有测试没有直接 method + partition 或 API assembly 覆盖；review 的批准边界是 static review、merge tree 与 CI，不能据此作 + broad compatibility/E2E 声明。^[PR #6609] ^[PR #6707, merged 2026-09-02] - 已知边界：同一 assembled app 的 upstream `GET /health` 仍先于 Omni health route 注册，当前 first-match dispatch 会遮蔽 Omni 的 diffusion fallback/JSON handler；测试 fake upstream 也未注册 该 sibling route。该 review thread 在本提交获批时明确 defer，不能据 profiler 修复声称所有重复 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md index 4e7e3d884..25074b5ab 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-upstream-compat.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, "PR #5036", "PR #6642", "PR #6773", vllm_omni/config/endpoint_policy.py] +sources: ["PR #5976", "PR #5957", vllm_omni/engine/stage_engine_startup.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/utils.py, vllm_omni/request.py, tests/engine/test_stage_engine_startup_cache_env.py, tests/config/test_endpoint_policy.py, "PR #5036", "PR #6642", "PR #6773", "PR #6707", vllm_omni/config/endpoint_policy.py] confidence: high --- @@ -41,14 +41,17 @@ confidence: high - 禁止：维护与 vLLM streamer 分离的本地 scheme allowlist；将对象存储 URI 当作 HF repo id 下载；让 bucket 或组织路径段参与模型名称匹配。 - 验收：参数化 mock 测试覆盖 `s3://`、`gs://` 及上游支持的 `az://`，断言 HF 下载未调用且返回值未改变；同时覆盖本地路径、HF repo id 和欺骗性 bucket/组织段匹配，确认入口语义没有回归。resolved HF cache snapshot 的 repo-name recovery 已随 #6642 的 revert 不再是当前合同。^[PR #5036] ^[PR #6642] -## SERV-7d — upstream error-response helper 必须按能力边界兼容两种布局 +## SERV-7d — error-response import 必须遵循各 caller 的已验证 owner - 触发：pinned vLLM 移动 `create_error_response`，或 endpoint policy / API server 增加直接 caller。 -- 强制：两个 caller 都先从新版 package owner `vllm.entrypoints.serve` 导入；只有该 import 抛 - `ImportError` 时才回退旧布局 `vllm.entrypoints.serve.utils.error_response`。两处必须保持同一顺序， - 使 endpoint rejection 与 engine error 都继续产生 upstream structured response。 -- 禁止：捕获任意 `Exception` 后伪装成版本兼容；在不同 caller 固定不同路径；因 import 成功就宣称 - 跨版本 API/JSON parity。该 shim 只覆盖这一个 symbol 的两种已知位置。 -- 验收：分别在仅提供新版 re-export、仅提供旧 module、两者都缺失的 import fixture 下加载两个 - caller，并验证 400/error response shape。PR #6773 没有新增自动化回归或硬件执行证据；其 diff - 只证明双 caller 的 import fallback，不能扩展为完整 vLLM 版本兼容声明。^[PR #6773] +- 强制：`endpoint_policy.py` 直接从 pinned owner + `vllm.entrypoints.serve.exception_handling.error_response` 导入 `create_error_response`，不设 fallback； + `api_server.py` 则先从 package root `vllm.entrypoints.serve` 导入，只在 `ImportError` 时回退 + `vllm.entrypoints.serve.utils.error_response`。这是两个 caller 各自的 source-authoritative 合同， + 不要求共享 fallback 顺序。 +- 禁止：把 endpoint-policy 的直接模块 import 改回 package-root fallback；捕获任意 `Exception` 伪装 + 成兼容；或以这两个 import 成功声称跨版本 API/JSON parity。 +- 验收：在此 pinned revision，直接导入 relocated module，并运行 + `tests/config/test_endpoint_policy.py`；API-server 路径仍仅为其 `ImportError` fallback 的静态合同。 + PR 描述报告该 suite `4 passed`，批准 review 的边界是 static review、merge tree 与 CI，且明确未执行 + untrusted fork code；因此不能扩展为完整 vLLM 版本兼容或端到端声明。^[PR #6707, merged 2026-09-02] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 51c6abc1e..5a29b0c13 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ca6b4e21`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 63ed0fde`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from b9a88bf5 \ - --to ca6b4e21 \ + --from ca6b4e21 \ + --to 63ed0fde \ --repo \ --mode report-only ``` From 03d991931f93c788bef3bb933d2d647a2ad3873f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:40:03 +0800 Subject: [PATCH 436/465] knowledge(vllm-omni): sync 376b28ac (PR #6920) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../repos/vllm-omni/models/cosmos3/_index.md | 14 ++++++++--- .../repos/vllm-omni/models/cosmos3/rules.md | 24 ++++++++++++++++++- 11 files changed, 46 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index b7ff9449b..cd9079199 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ca6b4e21127ff3f36530438a42948fe17117d412 + previous_audited_sha: 63ed0fdef4e6e9bdd866ea418ad9cf65e498c67a audited_ref: main - audited_sha: 63ed0fdef4e6e9bdd866ea418ad9cf65e498c67a + audited_sha: 376b28aced77453c255f3a1d92d9f7c5149c4699 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4d9c74372..85e9f8ba9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -63ed0fdef4e6e9bdd866ea418ad9cf65e498c67a`. The canonical machine baseline +376b28aced77453c255f3a1d92d9f7c5149c4699`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 740e63f91..a339832f5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 63ed0fde` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 376b28ac` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index aa75d50b9..3c24aa279 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 63ed0fde` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 376b28ac` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 4d60ca60f..15bfd24bb 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 63ed0fde` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 376b28ac` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e9b7bd728..0220f5dc7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 63ed0fde` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 376b28ac` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index f042c0aea..518f67a12 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 63ed0fde` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 376b28ac` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e90af203d..58ee080d9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 63ed0fde` 验证存在 +- 源码校验：以上路径均已在 `main @ 376b28ac` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5a29b0c13..2137c2378 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 63ed0fde`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 376b28ac`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from ca6b4e21 \ - --to 63ed0fde \ + --from 63ed0fde \ + --to 376b28ac \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosmos3/_index.md b/knowledge/repos/vllm-omni/models/cosmos3/_index.md index aa09cfa9e..d4a07f0c8 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/_index.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/_index.md @@ -1,10 +1,10 @@ --- title: "Cosmos3" created: 2026-07-20 -updated: 2026-09-02 +updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6920", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py] confidence: high --- @@ -31,6 +31,14 @@ transformer 还要维护自己的 layerwise-offload block 声明。architecture 变体专有不变量与描述直达源码入口见 [rules](rules.md#direct-代码快速入口)。新模型通用验证见 [model validation](../../review/guides/model-validation.md)。 +## Distributed VAE multi-chunk transfer + +当 transfer 用 distributed VAE 解码多 chunk 视频时，rank 0 是完整 video/control output 的唯一 +assembler；每个非最终 chunk 只同步下一 chunk 条件所需的 decoded tail 给所有 rank。非输出 rank +保留自己的最终 decoded output 和 metadata，不拼接全量结果。该路径及其 executor gate、overlap shape +和 session-state guard 由 `COSMOS-6b` 约束；不要把一次 4×GB300 回归外推为通用 multi-GPU、parity、 +质量或性能支持。 + ## ROCm evidence scope ROCm recipe 的 MI350X latency/显存只来自 `b3f4fbf9` 上单卡 gfx950、Nano T2I/T2V、 @@ -47,5 +55,5 @@ request 的 denoise 生命周期内使用，不提供跨请求记忆或并发安 ## 什么时候查这里 -- 审查 Cosmos3 Edge/Distilled scheduler、RNG、guidance、offload 或 capability claim。 +- 审查 Cosmos3 Edge/Distilled scheduler、RNG、guidance、offload、distributed-VAE multi-chunk transfer 或 capability claim。 - 同一问题影响多个 diffusion 模型时返回 [Diffusion rules](../../components/diffusion/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/cosmos3/rules.md b/knowledge/repos/vllm-omni/models/cosmos3/rules.md index 995b76fbf..2b1008121 100644 --- a/knowledge/repos/vllm-omni/models/cosmos3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosmos3/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325", "PR #6913"] +sources: ["PR #4657", "PR #5001", "PR #5634", "PR #6049", docs/features/session_state_manager.md, recipes/cosmos3/Cosmos3-Nano.md, vllm_omni/diffusion/models/cosmos3/, vllm_omni/model_extras/cosmos3.py, vllm_omni/model_extras/registry.py, vllm_omni/experimental/world_models/adapters/state_cosmos3_adapter.py, vllm_omni/platforms/rocm/platform.py, tests/diffusion/models/cosmos3/test_session_memory_equivalence.py, tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py, "PR #6107", "PR #5614", "PR #6325", "PR #6913", "PR #6920"] confidence: high --- @@ -24,6 +24,7 @@ confidence: high | online/offload/HSDP/VAE parallel 支持声明 | COSMOS-3a | capability 文档/recipe → 对应公开入口与实现路径 | | ROCm、MI350X、AITER、latency/peak memory | COSMOS-3b | recipe 的 measurement commit/protocol → `platforms/rocm/platform.py` backend gate → 目标配置复测 | | formatted final prompt、`prompt_suffix`、rank-zero log level | COSMOS-6a | `diffusion/models/cosmos3/pipeline_cosmos3.py::Cosmos3OmniDiffusersPipeline` | +| multi-chunk transfer、distributed VAE decode、overlap broadcast、rank-zero assembly | COSMOS-6b | `diffusion/models/cosmos3/pipeline_cosmos3.py::{_transfer_vae_executor,_sync_transfer_overlap,_forward_transfer}`；`tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py` | 若描述只写 Cosmos3，先从 registry 的 class key 进入 pipeline；只有命中共享 RNG、graph 或 offload 机制时再加 [Diffusion owner](../../components/diffusion/rules.md)。 @@ -137,3 +138,24 @@ confidence: high - 验收：rank-zero test 用含 sentinel 与 suffix 的 prompt 断言最终字符串和 generation 输入不变，且 DEBUG 才有该条记录；non-rank control 无此 emission。target 未新增 log-level test，证据仅为作者 报告的 CPU unit/manual check，不能外推为其它模型、日志配置或生产运行结论。^[PR #6913] + +## COSMOS-6b — multi-chunk transfer 必须同步分布式 VAE overlap + +- 触发：修改 Cosmos3 transfer 的 chunk 边界、`previous_output` 条件帧、VAE decode，或 distributed + VAE 的 rank 输出/collective 路径。 +- 强制：只有 `vae.distributed_executor` 存在且 `vae.is_distributed_enabled` 可调用并返回 true 时才启用 + distributed VAE 路径。每个非最终 chunk 的 overlap 必须是 + `min(num_conditional_frames, chunk_frames)`；overlap 小于等于零时返回 `None`，且不调用 broadcast。 + active executor 下所有 rank 都对每个需要同步的 overlap 调用 `broadcast_tensor`：rank 0 从已 decode + output 尾部 slice 并 `contiguous()`，其他 rank 分配与 reference video 同 batch/channel/height/width、但 + frame 维为 overlap 的 receive tensor，且使用 `self.device` 和 `vae.dtype`。仅 rank 0 拼接完整 video/control output；其他 + rank 在 collective 完成后返回本地最终 decoded output 与 metadata。 +- 禁止：仅因 executor 属性存在就进入分布式路径；让任一 rank 跳过应参与的 broadcast；broadcast 整段 + output、从非 rank-zero 的空 output slice overlap，或在非 rank-zero 拼接 full/control output。该同步 + 不得放宽 `COSMOS-2c` 的 session-state guard：flag 开启时 `diffuse_transfer()` 仍须 fail closed，即使 + `_bde_kv_state` 非空。 +- 验收：mock executor 覆盖 enabled/callable gate、rank-zero tail slice、nonzero-rank receive shape/dtype/device、 + 每个 non-final chunk 的 collective 以及 nonpositive-overlap 的 no-broadcast；仅有界的真实回归是一次 + 4×GB300 multi-chunk transfer（Ulysses 4、VAE patch parallelism 4、tiling，1280×720、101 frames、24 FPS、 + 53-frame chunks、5 conditional frames、2 steps），确认完成且 metadata/video 为 101 frames at 24 FPS。 + 该证据不构成广泛 parity、质量或性能声明。^[PR #6920] From ff01af19c7cb339a8eb3b197e00b8bbed2453d77 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:46:45 +0800 Subject: [PATCH 437/465] knowledge(vllm-omni): sync 8f284b34 (PR #6786) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/parallelism.md | 4 +++- .../diffusion/rules-system-runtime.md | 15 ++++++++++++++- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/models/boogu-image/_index.md | 4 ++-- .../repos/vllm-omni/models/boogu-image/rules.md | 17 ++++++++++++++++- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 13 files changed, 47 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cd9079199..bd72ffccb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 63ed0fdef4e6e9bdd866ea418ad9cf65e498c67a + previous_audited_sha: 376b28aced77453c255f3a1d92d9f7c5149c4699 audited_ref: main - audited_sha: 376b28aced77453c255f3a1d92d9f7c5149c4699 + audited_sha: 8f284b343af4904225088ff8a9ffab4d43768568 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 85e9f8ba9..89729b395 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -376b28aced77453c255f3a1d92d9f7c5149c4699`. The canonical machine baseline +8f284b343af4904225088ff8a9ffab4d43768568`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a339832f5..28e2c15b4 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 376b28ac` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8f284b34` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3c24aa279..5452d66fd 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 376b28ac` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 8f284b34` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/parallelism.md b/knowledge/repos/vllm-omni/components/diffusion/parallelism.md index dcb6022fb..b2dcc8257 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/parallelism.md +++ b/knowledge/repos/vllm-omni/components/diffusion/parallelism.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-05 type: guide tags: [vllm-omni, components, diffusion] -sources: [docs/design/feature/tensor_parallel.md, docs/design/feature/cfg_parallel.md, docs/design/feature/vae_parallel.md, "PR #6340", vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/attention/parallel/ulysses.py] +sources: [docs/design/feature/tensor_parallel.md, docs/design/feature/cfg_parallel.md, docs/design/feature/vae_parallel.md, "PR #6340", "PR #6786", vllm_omni/diffusion/distributed/a2a_permute.py, vllm_omni/diffusion/distributed/cfg_parallel.py, vllm_omni/diffusion/attention/parallel/ulysses.py] --- # Diffusion 并行策略总览 @@ -26,5 +26,7 @@ sources: [docs/design/feature/tensor_parallel.md, docs/design/feature/cfg_parall advanced-UAA 与 AllGather 不走该 path；strict Ulysses+Ring 的 Ulysses leg 可用。workspace/capture/shutdown 与硬件 parity 门禁见 [DIFF-4x](rules-system-runtime.md#diff-4x-symmmem-ulysses-a2a-只能显式启用并保持-workspace-生命周期闭合)， 配置投影见 [VOMNI-CFG-1q](../configuration/rules-diffusion-parallel-transport.md#vomni-cfg-1q-symmmem-ulysses-transport-开关必须完整投影且默认关闭)。^[PR #6340] +- CFG parallel state 的未初始化 serial-1 fallback、已初始化错误传播以及 two/N-branch rank + 分派，见 [DIFF-4y](rules-system-runtime.md#diff-4y-cfg-parallel-state-未初始化时串行初始化错误必须可见)。 - 并行度 × 设备容量的启动验收硬规则在 [Model Executor 规则](../model-executor/rules.md)。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md index 815dbfff7..238972ef7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-system-runtime.md @@ -4,7 +4,7 @@ created: 2026-09-03 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", "PR #6814", "PR #6563", "PR #5716", vllm_omni/diffusion/attention/, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/attention/parallel/ring_kernels.py, vllm_omni/diffusion/diffusion_kv/, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/npu/platform.py, tests/diffusion/diffusion_kv/, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_ulysses_uaa.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] +sources: ["PR #5255", "PR #5344", "PR #5543", "PR #5838", "PR #6094", "PR #6102", "PR #6385", "PR #6340", "PR #6714", "PR #6814", "PR #6563", "PR #5716", "PR #6786", vllm_omni/diffusion/attention/, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/attention/parallel/ring_kernels.py, vllm_omni/diffusion/diffusion_kv/, vllm_omni/diffusion/distributed/cfg_parallel.py, vllm_omni/diffusion/distributed/parallel_state.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/npu/platform.py, tests/diffusion/diffusion_kv/, tests/diffusion/distributed/test_cfg_parallel.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_ulysses_uaa.py, "PR #5491", "PR #5194", "vllm_omni/diffusion/data.py", "vllm_omni/diffusion/utils/hf_utils.py"] confidence: high --- @@ -93,6 +93,19 @@ confidence: high - 禁止：把 flag 当作默认 transport、静默 fallback；按历史 shape/dtype 无界缓存 allocation；跨 stream 重用 staging buffer；capture 中隐式分配/扩容；把任意 strided tensor 当作 copy-engine eligible，或把 CPU/mock workspace 测试、PR benchmark 写成数学/输出 parity、普遍速度或跨硬件收益。 - 验收：覆盖 CLI/deploy/default-stage 的显式 flag 透传与默认 false、eligible strict path 的 init build、UAA/AllGather/noneligible layout 不调用该 path、strict Ulysses+Ring 的 eligible Ulysses leg、peak workspace 复用/collective growth、capture growth fail、single-stream reject 与 shutdown release；copy op 必须拒绝 device/dtype/rank/shape/output-contiguity/inner-row/overlap 不匹配。当前没有新增 upstream unit；仅 B300 上 row-strided BF16 bitwise copy、四-rank forward/reverse parity 和四-GPU H3 smoke 证据，不能外推为一般硬件或性能结论。^[PR #6340] ^[PR #6814] +## DIFF-4y — CFG parallel state 未初始化时串行，初始化错误必须可见 + +- 触发：修改 `cfg_parallel`、CFG process-group 初始化/查询、正负 guidance branch 的 rank + 分派，或其 runner 调用链。 +- 强制：CFG parallel state 未初始化时，所有 rank 必须走 serial-1 路径；state 已初始化后， + group 查询、world-size/rank 或 collective 的错误必须原样传播，不能将错误降级成 serial。 + 已初始化 group 的 two-branch 与 N-branch 路径都必须按该 group 的 rank/world-size 分派，不能 + 以默认 global parallel state 猜测 CFG 拓扑。 +- 禁止：把“group 不存在”和已初始化 group 的查询/通信失败混为同一 fallback；只实现 two-branch + 而让 N-branch 静默串行、重复或遗漏 branch；在调用方吞掉初始化后的错误。 +- 验收：以 mock/distributed test 覆盖未初始化 → serial-1、已初始化 group 的错误传播、two-branch + rank 分派与 N-branch rank 分派；测试只证明这些 CFG state/path 合同，不外推为多节点吞吐或模型质量。^[PR #6786] + ## DIFF-4g — Ring GQA 只在通信后扩展，Hybrid Ulysses 不把 padding 当真 head - 触发：修改 Ring SDPA、GQA/MQA head 数、LSE accumulation、Ulysses all-to-all 或 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 15bfd24bb..a69ae416c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 376b28ac` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8f284b34` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0220f5dc7..a45e05bfb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 376b28ac` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8f284b34` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 518f67a12..c676f9d26 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 376b28ac` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8f284b34` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 58ee080d9..88ead4259 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 376b28ac` 验证存在 +- 源码校验：以上路径均已在 `main @ 8f284b34` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/boogu-image/_index.md b/knowledge/repos/vllm-omni/models/boogu-image/_index.md index 6ced4cd51..991d63b9a 100644 --- a/knowledge/repos/vllm-omni/models/boogu-image/_index.md +++ b/knowledge/repos/vllm-omni/models/boogu-image/_index.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #6701", "PR #6571", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/diffusion/models/boogu_image/, vllm_omni/entrypoints/utils.py] +sources: ["PR #6701", "PR #6571", "PR #6786", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/diffusion/models/boogu_image/, vllm_omni/entrypoints/utils.py] confidence: high --- @@ -51,4 +51,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| revision/component loader、Edit-Turbo checkpoint 或 real RoPE/platform selection | [Boogu rules](rules.md) | +| revision/component loader、Edit-Turbo checkpoint、real RoPE/platform selection，或 Edit CFG branch/rank/VAE decode 合同 | [Boogu rules](rules.md#boogu-1c-edit-cfg-分支rank-分派与-vae-decode-必须保持语义闭合) | diff --git a/knowledge/repos/vllm-omni/models/boogu-image/rules.md b/knowledge/repos/vllm-omni/models/boogu-image/rules.md index e32b79607..724c444fc 100644 --- a/knowledge/repos/vllm-omni/models/boogu-image/rules.md +++ b/knowledge/repos/vllm-omni/models/boogu-image/rules.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #6701", "PR #6571", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py, vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py, vllm_omni/entrypoints/utils.py, tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py, tests/diffusion/models/boogu_image/test_boogu_image_transformer.py, tests/entrypoints/test_utils.py, tests/e2e/online_serving/test_boogu_image_edit.py] +sources: ["PR #6701", "PR #6571", "PR #6786", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py, vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py, vllm_omni/entrypoints/utils.py, tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py, tests/diffusion/models/boogu_image/test_boogu_image_transformer.py, tests/entrypoints/test_utils.py, tests/e2e/online_serving/test_boogu_image_edit.py] confidence: high --- @@ -26,3 +26,18 @@ confidence: high - 强制：`(cos,sin)` 各为 `B,S,D`；per-axis tables 用 `use_real`/`repeat_interleave_real` adjacent-pair frequency，分别 gather/concat cos、sin。MPS CPU-gather indices 后返回原 device；NPU/MPS/no-FP64 用 FP32，否则 FP64。adjacent pairs 在 FP32 以 `even*cos-odd*sin`/`even*sin+odd*cos` rotate 后 cast x dtype；instruction/image refinement、double/single stream 和 pipeline 全部传此表示。 - 禁止：Ascend complex path、swapped trig/pair layout，或 bitwise/broad platform/perf claims。 - 验收：old complex reference assert-close，加 MPS/NPU selection 和 pipeline propagation；E2E/perf 仅作者报告。^[PR #6571] + +## BOOGU-1c — Edit CFG 分支、rank 分派与 VAE decode 必须保持语义闭合 + +- 触发：修改 Boogu-Image Base/Edit sampling、single/double-stream transformer、CFG parallel + branch construction/combination，或 CUDA VAE decode。 +- 强制：Base 与 single-stream Edit 都构造两条 branch。double-stream Edit 必须按 + `[positive+reference, negative+reference, negative-no-reference]` 构造；CFG2 时 rank 0 消费 + `[0,2]`、rank 1 消费 `[1]`，CFG3 时每 rank 消费一条。合并严格按 + `pwr + (text - 1) * (pwr - nwr) + (image - 1) * (nwr - uncond)`；CFG 关闭时每个 rank + 只执行 positive branch。CUDA VAE encode/decode 优先 `EFFICIENT_ATTENTION`，再以 `MATH` fallback。 +- 禁止：重排或省略 Edit 的 reference-aware negative branch；把 CFG2/CFG3 rank 负载混用；CFG-off + 仍执行 negative branch；以 TP、SP、HSDP、cache 或 offload 组合宣称该合同已支持。 +- 验收：有界测试覆盖 Base/single Edit 两分支、double-stream Edit 三分支、CFG2 的 + `rank0=[0,2]`/`rank1=[1]`、CFG3 一 rank 一 branch、精确 guidance 合并式、CFG-off 每 rank + positive-only，以及 CUDA VAE efficient→math fallback；不将这些测试外推为 TP/SP/HSDP、cache、offload 或通用性能结论。^[PR #6786] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 2137c2378..ad094cd0d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 376b28ac`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8f284b34`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 63ed0fde \ - --to 376b28ac \ + --from 376b28ac \ + --to 8f284b34 \ --repo \ --mode report-only ``` From 18c5dbd955688965c764b542d11e3f6806a41802 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:51:14 +0800 Subject: [PATCH 438/465] knowledge(vllm-omni): sync 4927fd54 (PR #6268) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/diffusion/rules-lora.md | 28 +++++++++++++++++-- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 10 files changed, 38 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bd72ffccb..a73a3b1a6 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 376b28aced77453c255f3a1d92d9f7c5149c4699 + previous_audited_sha: 8f284b343af4904225088ff8a9ffab4d43768568 audited_ref: main - audited_sha: 8f284b343af4904225088ff8a9ffab4d43768568 + audited_sha: 4927fd5412f22098a0f8aa8aa455a538d53e433c # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 89729b395..c99fdb40e 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8f284b343af4904225088ff8a9ffab4d43768568`. The canonical machine baseline +4927fd5412f22098a0f8aa8aa455a538d53e433c`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 28e2c15b4..e442bf5b6 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8f284b34` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4927fd54` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 5452d66fd..e4dd3dfd7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8f284b34` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 4927fd54` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md index 54d77f48c..a026c50e9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-lora.md @@ -1,10 +1,10 @@ --- title: "Diffusion LoRA 规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/lora/layers/base_linear.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/diffusion/lora/test_lora_manager.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070", "PR #6476", "PR #6550", vllm_omni/diffusion/models/minimax_h3/lora.py] +sources: ["PR #2783", docs/user_guide/diffusion/lora.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/lora/loader.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/lora/layers/base_linear.py, vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py, vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py, vllm_omni/diffusion/utils/tf_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, tests/diffusion/lora/test_loader.py, tests/diffusion/lora/test_lora_manager.py, tests/entrypoints/test_async_omni_diffusion_config.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6070", "PR #6476", "PR #6550", vllm_omni/diffusion/models/minimax_h3/lora.py, "PR #6268", benchmarks/kernels/benchmark_diffusion_lora_expand.py, tests/diffusion/lora/test_base_linear.py] confidence: high --- @@ -20,6 +20,7 @@ confidence: high | `--lora-backend`、startup path/scale、deploy/CLI projection | `DIFF-2l` | `entrypoints/cli/serve.py` → `async_omni_engine.py`/stage config → `OmniDiffusionConfig` → `DiffusionWorker.init_lora_manager` | | distilled merge、key conversion、packed QKV、alpha、unload | `DIFF-2m` | `diffusion/lora/loader.py` → pipeline mixin → transformer parameter | | Wan dual transformer、high/low-noise file order、partial load | `DIFF-2n` | `WanLoraLoaderMixin` → `get_transformer_from_pipeline(name)` → `transformer`/`transformer_2` | +| LoRA linear active output slices、inference expand accumulation、autograd/dtype fallback | `DIFF-2ab` | `diffusion/lora/layers/base_linear.py::DiffusionBaseLinearLayerWithLoRA.apply` | | legacy dynamic adapter hook、packed/stacked binding、DLO sidecar residency、activation rollback | `DIFF-2x` | `DiffusionLoRAManager::{_load_adapter,_bind_adapter_weights,_activate_adapter}` → model pipeline hook | | 审查组 | 什么时候触发 | 规则 ID | @@ -95,6 +96,29 @@ confidence: high - 禁止：为 phase adapter 增加第二个 resident Transformer 或 LTX 专用环境变量；允许 request/static LoRA 与内部 phase adapter 组合；在非 BF16 或量化 base 上声称 layer-fused 可用；对缺失配对、未映射模块或不匹配 shard 静默跳过。 - 验收：覆盖官方 key mapping、缺失 A/B、unmapped/duplicate target、dynamic 与 fused 精确算术、Row/Column/QKV 本地切片、base 权重不变、phase 进入/退出和 finalize 时机；同时验证 generic `model_paths` sidecar 覆盖与 model-root/Hub fallback。^[PR #5500] +## DIFF-2ab — LoRA linear expand 只能在原输出 slice 上作受限的 inference 累加 + +- 触发：修改 `DiffusionBaseLinearLayerWithLoRA.apply()` 的 stacked/packed LoRA shrink-then-expand + 路径、其 active-slice 跳过、输出 slice，或为该路径做 kernel/temporary-allocation 优化。 +- 强制：保留 fully-sharded LoRA 与 `tp_size > 1` 的 `NotImplementedError`，直到 shrink 与 expand + 之间实现所需 all-gather。将 `x`/base output 展平后，按 `output_slices` 的累计 `offset` 处理每一 + slice；inactive 或空 A/B slice 也必须推进 offset，且 slice metadata 与 A/B stacks 长度不一致必须 + 失败。对 active nonempty slice 先算 `buffer = x_flat @ A.t()`，再以 + `torch.addmm(y_slice, buffer, B.t(), out=y_slice)` 直接累加 expand，**仅当** grad mode disabled 且 + `y_slice`、`buffer`、`B` 三者 dtype 完全相同。 +- 强制：grad mode enabled 或任一上述 dtype 不同，必须保留 functional + `y_slice[:] = y_slice + buffer @ B.t()`，以维持 autograd 和 output dtype 语义；最终返回的 shape + 必须恢复为 base output 的原 shape。 +- 禁止：用 `out=` 路径绕过 autograd/dtype fallback，漏掉 inactive/empty slice 的 offset，或把目前的 + fully-sharded+TP rejection 伪称为支持。不要新增 stride/contiguity predicate 来选择该 fast path，也 + 不要把此处的 synthetic microbenchmark 扩大为任意 strided output、完整 diffusion model 或 server + throughput 的性能承诺。 +- 验收：CPU regression 覆盖 packed multi-slice 的数值结果与每个 active slice 的 in-place `addmm`、 + inactive/empty slice 后的 offset、fully-sharded+TP rejection、autograd backward 时不调用 `out=`， + 以及 mixed output/LoRA dtype 的 functional fallback。性能证据仅可用固定 shape/dtype/rank、warmup + 与 iteration 的 CUDA microbenchmark；PR evidence 是 H100 上 BF16 packed synthetic expand 的 + latency/extra-allocation 和 numerical-parity 测量，不替代上述 correctness cases。^[PR #6268] + ## DIFF-2x — legacy dynamic manager 的模型专用 loader 与 activation 必须 fail-closed 且事务化 - 触发：修改 `DiffusionLoRAManager` 的 request-time adapter load/bind、模型专用 loader hook、packed/stacked target，或 DLO 下 dynamic sidecar 的 device placement/reallocation。 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a69ae416c..247f8fbf1 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8f284b34` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4927fd54` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index a45e05bfb..9822bc326 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8f284b34` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4927fd54` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c676f9d26..c830362e9 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8f284b34` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4927fd54` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 88ead4259..4fe63ba9d 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8f284b34` 验证存在 +- 源码校验：以上路径均已在 `main @ 4927fd54` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index ad094cd0d..6dcbfa921 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8f284b34`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4927fd54`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 376b28ac \ - --to 8f284b34 \ + --from 8f284b34 \ + --to 4927fd54 \ --repo \ --mode report-only ``` From 896a62f0e590c8dee23208e5db2795fae41e3fc8 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:54:25 +0800 Subject: [PATCH 439/465] knowledge(vllm-omni): sync b5e898f2 (PR #4048) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 3 ++- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../repos/vllm-omni/models/hunyuan-image3/_index.md | 2 +- .../vllm-omni/models/hunyuan-image3/architecture.md | 5 ++++- .../repos/vllm-omni/models/hunyuan-image3/rules.md | 11 +++++++++-- 12 files changed, 27 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a73a3b1a6..a7c51280d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8f284b343af4904225088ff8a9ffab4d43768568 + previous_audited_sha: 4927fd5412f22098a0f8aa8aa455a538d53e433c audited_ref: main - audited_sha: 4927fd5412f22098a0f8aa8aa455a538d53e433c + audited_sha: b5e898f295382b0d3457da95d0901dda3701ebb8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c99fdb40e..0180c8033 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4927fd5412f22098a0f8aa8aa455a538d53e433c`. The canonical machine baseline +b5e898f295382b0d3457da95d0901dda3701ebb8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e442bf5b6..e19939131 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4927fd54` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b5e898f2` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e4dd3dfd7..9ca67bac1 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4927fd54` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ b5e898f2` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -42,6 +42,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | | video/audio mux、DLO DP wave、result queue、async pump、SHM ownership、async-output wait 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | +| HunyuanImage3 Distil CFG、MeanFlow special-token / scheduler-r 合同 | [HunyuanImage3 HY3-8d](../../models/hunyuan-image3/rules.md) | | diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | | request-wave admission coalescing、stable window、deadline 与 finite config | [admission wait rules](rules-admission-wait.md) | | paged KV/cache 预算、native/backend/platform 闭环、GQA/Ring/Ulysses layout、FlashInfer plan、能力 metadata | [paged cache 与系统运行时规则](rules-system-runtime.md) | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 247f8fbf1..91242d5a8 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4927fd54` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b5e898f2` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9822bc326..609a62e31 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4927fd54` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b5e898f2` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c830362e9..d01652974 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4927fd54` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b5e898f2` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 4fe63ba9d..b626565e3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4927fd54` 验证存在 +- 源码校验：以上路径均已在 `main @ b5e898f2` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6dcbfa921..d5fbdd2f3 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4927fd54`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b5e898f2`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 8f284b34 \ - --to 4927fd54 \ + --from 4927fd54 \ + --to b5e898f2 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md index b40d21bfc..6e3c6ab8d 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/_index.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: index tags: [vllm-omni, models, hunyuan-image3] -sources: ["PR #5541", "PR #6563", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py] +sources: ["PR #5541", "PR #6563", "PR #4048", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py] --- # HunyuanImage3 diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md index 382c5f674..c01db287a 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: architecture tags: [vllm-omni, models, hunyuan-image3] -sources: ["PR #6563", vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py] +sources: ["PR #6563", "PR #4048", vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py] --- # vLLM-Omni 核心架构 @@ -49,6 +49,9 @@ sources: ["PR #6563", vllm_omni/model_executor/models/hunyuan_image3/hunyuan_ima 和 metadata。模型只交 32Q/8KV 与 `full_attn_spans`，runner 激活 request-mode prefill/denoise rows：首步 覆盖整 allocation，之后只更新 prefix-offset target；dense prompt KV 在 paged forward 前清除。此范围不含 `denoise_step`、AR KV、negative CFG、Ring、AllGather 或 cross-request reuse。^[PR #6563] +- Distil conditioning：`cfg_distilled` 和 `use_meanflow` 分别追加 guidance / next-timestep-r special token；外层 + pipeline 加载对应 embedding。embedded CFG 固定单 row，MeanFlow 最后一步 `r=0`；layout、step execution 与 + AR-KV reuse 以 `num_special_tokens` 和 rebased scatter indices 保持同一 token contract。^[PR #4048] - Tokenizer：`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py` - VAE wrapper：`vllm_omni/diffusion/models/hunyuan_image3/autoencoder.py` - MoE dispatch：`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py` diff --git a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md index 9cbf69f02..30a414137 100644 --- a/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md +++ b/knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md @@ -1,10 +1,10 @@ --- title: "HunyuanImage3 开发规则" created: 2026-07-13 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, hunyuan-image3] -sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", "PR #6306", "PR #6102", "PR #6563", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py] +sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/diffusion/models/hunyuan_image3/prompt_utils.py, vllm_omni/model_extras/hunyuan_image3.py, vllm_omni/model_extras/registry.py, "PR #6094", "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", "PR #6306", "PR #6102", "PR #6563", "PR #4048", vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py, vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_tokenizer.py, vllm_omni/diffusion/models/hunyuan_image3/request_layout.py, tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py, tests/diffusion/models/hunyuan_image3/test_image_kv_cache_manager.py] --- # HunyuanImage3 开发规则 @@ -143,6 +143,13 @@ sources: [incidents/painterly/_index.md, hf-alignment-pitfalls.md, vllm_omni/dif - 禁止：在非 CUDA 平台导入或强制执行 NVIDIA block；把 Triton 可导入等同于所有平台支持；改变 `in_layers`/`out_layers` 等 checkpoint-facing module key；移除逐卷积内存保护后仍让 tiling-disabled 的高分辨率请求无条件执行；或在可能重叠的 forward 中修改进程级 cuDNN 标志。 - 验收：分别以 native/eager 与 CUDA/NVIDIA 实现覆盖 VAE、DiT 的真实 shape、dtype 和输出误差；验证旧 checkpoint 的 `state_dict` 加载与 key parity；对启用 tiling、关闭 tiling及直接 VAE 入口执行高分辨率容量/失败行为测试，并在并发 worker 或明确串行证明下验证 cuDNN 配置与恢复；性能数字只在固定模型、硬件、并行拓扑和 workload 下复测。^[PR #6306] +- **HY3-8d — Distil CFG 与 MeanFlow 是独立的 HunyuanImage3 diffusion 合同** + - 触发：修改 `cfg_distilled`、`use_meanflow`、Hunyuan image layout/tokenizer、denoise/step execution、CFG、AR KV reuse，或相关 checkpoint loader。 + - 强制：两个 flag 独立决定 ``、`` placeholder 和对应 embedding；layout 的序列化、tokenizer output、prepared/step kwargs、CFG-parallel split 与 AR-KV truncate 必须完整携带并重基 scatter index。special-prefix 数量由 `num_special_tokens` 统一计算，decode final layer 从该数量之后取 latent，不能假定只有 timestep token。`guidance_emb` 与 `timestep_r_emb` 由外层 `HunyuanImage3Pipeline` 创建、放到 execution device 并加载权重，不由 transformer-core loader 静默跳过。 + - 强制：`cfg_distilled=True` 时 `cfg_factor=1`，guidance 以 `guidance_scale * 1000` 写入 embedding；不得构造 negative prompt/prefill、外部 CFG combine 或 CFG parallel。`use_meanflow=True` 时每个 denoise step 注入 scheduler 的下一 timestep `r`；最后一个 step 的 `r` 必须为零。两项 flag 可以单独开启，不能把 MeanFlow-only 路径误当成 CFG-distil。 + - 禁止：用 `x[:, 1:, :]` 或固定 special-token 个数切 latent；只在 normal loop 而漏掉 step execution；AR prefix 截断后保留绝对 guidance/timestep-r index；漏传任一 embedding/scatter field；把 transformer 的 non-model skip list 当作外层 embedding 的加载路径。 + - 验收：CPU 测试覆盖 guidance/timestep-r 四个 kwargs、1/2/3 special-prefix decode、AR-KV rebasing、MeanFlow-only CFG-parallel split，以及 step execution 的 single-row CFG-distil 和每-step `r`。准确性证据固定模型 artifact、BF16/硬件拓扑、seed、steps、guidance、prompt/reference 和各项 threshold；只报告实际运行的 online/offline/quant case，未运行的 quant 不得外推为通过。^[PR #4048] + - **HY3-5h — paged GQA 必须保留压缩 K/V，dense 路径保持扩展** - 触发：修改 HunyuanImage3 的 `ImageKVCacheManager`、Q/K/V head layout，或在 dense 与 Scheduler-managed paged attention 之间切换 GQA 处理。 - 强制：HunyuanImage3 的 dense 路径继续把 K/V 从 8 个 KV heads 扩展到 32 个 Q heads；paged KV active 时必须保留原生压缩的 32Q/8KV 形状，由 native adapter/backend 消费 `num_heads=32` 与 `num_kv_heads=8`。 From ee9e42530dfeae453eac38cdbd68f5265c831442 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:57:32 +0800 Subject: [PATCH 440/465] knowledge(vllm-omni): sync 48ae412a (PR #6537) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 5 +-- .../serving/rules-session-lifecycle.md | 32 +++++++++++++++++ .../vllm-omni/components/serving/rules.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../repos/vllm-omni/models/mage-vl/_index.md | 35 +++++++++++++++++++ .../vllm-omni/models/mage-vl/architecture.md | 34 ++++++++++++++++++ .../repos/vllm-omni/models/mage-vl/rules.md | 32 +++++++++++++++++ 14 files changed, 148 insertions(+), 14 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/serving/rules-session-lifecycle.md create mode 100644 knowledge/repos/vllm-omni/models/mage-vl/_index.md create mode 100644 knowledge/repos/vllm-omni/models/mage-vl/architecture.md create mode 100644 knowledge/repos/vllm-omni/models/mage-vl/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a7c51280d..607a20bb2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4927fd5412f22098a0f8aa8aa455a538d53e433c + previous_audited_sha: b5e898f295382b0d3457da95d0901dda3701ebb8 audited_ref: main - audited_sha: b5e898f295382b0d3457da95d0901dda3701ebb8 + audited_sha: 48ae412a2f6feb3c097cbc915d6427a163e454e3 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 0180c8033..84b6cf129 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b5e898f295382b0d3457da95d0901dda3701ebb8`. The canonical machine baseline +48ae412a2f6feb3c097cbc915d6427a163e454e3`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e19939131..49c62f590 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b5e898f2` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 48ae412a` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 9ca67bac1..f1a9f7b0b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b5e898f2` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 48ae412a` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 91242d5a8..c329aafa7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b5e898f2` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 48ae412a` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 609a62e31..70301ab16 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b5e898f2` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 48ae412a` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d01652974..c6254897f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b5e898f2` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 48ae412a` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index b626565e3..a878e42ab 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b5e898f2` 验证存在 +- 源码校验：以上路径均已在 `main @ 48ae412a` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -33,8 +33,9 @@ sources: [] | upstream launcher shutdown、renderer warmup 与多 replica compile-cache 隔离 | [upstream 兼容规则](rules-upstream-compat.md) | | stage/replica 死亡、request-local failure、readiness/liveness 或驱逐后 cleanup | [fault-isolation rules](rules-fault-isolation.md) | | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | -| engine startup/shutdown、stage lifecycle、acknowledged abort 的 shutdown race、full-duplex、CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | +| engine startup/shutdown、stage lifecycle、acknowledged abort 的 shutdown race、duplex stage prewarm/CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | | duplex stage-keyed segment state、跨 stage boundary/metadata 隔离 | [duplex 编排规则](rules-duplex-orchestration.md) | +| adapter-owned full-duplex session、deferred cognition/gate、barge-in cancellation、response ownership、`to_thread` worker drain 或 close/flush/session cleanup | [session lifecycle 规则](rules-session-lifecycle.md) | | acknowledged abort 的 RPC ACK、queue/router shutdown race、cancellation cleanup 或 backpressure/timeout 保持 | [acknowledged abort 生命周期规则](rules-abort-lifecycle.md) | | assembled FastAPI route 覆盖、method/path ownership 或 request-body forwarding | [app assembly 规则](rules-app-assembly.md) | | stage config→EngineArgs projection、explicit `devices`、TP/local-DP/PP、replica layout 或 worker 创建前的 layout guard | [stage 启动与设备布局规则](rules-stage-startup.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-session-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-session-lifecycle.md new file mode 100644 index 000000000..afb4483f1 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/serving/rules-session-lifecycle.md @@ -0,0 +1,32 @@ +--- +title: "Serving session lifecycle 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, serving] +sources: ["PR #6537", vllm_omni/experimental/fullduplex/core/adapter.py, vllm_omni/experimental/fullduplex/core/runtime.py, vllm_omni/experimental/fullduplex/mage_vl/adapter.py, vllm_omni/experimental/fullduplex/mage_vl/serving/backend.py, vllm_omni/experimental/fullduplex/mage_vl/serving/server.py, tests/e2e/features/fullduplex/test_mage_vl_adapter.py, tests/e2e/features/fullduplex/test_mage_vl_serving.py] +confidence: high +--- + +# Serving session lifecycle 规则 + +## SERV-6h — deferred session cognition 必须保持 input responsiveness、response ownership、worker drain 与 cleanup + +- 触发：共享 `DuplexAdapter`/`DuplexRuntime` lifecycle、deferred cognition/gate、full-duplex barge-in、 + shared-GPU `asyncio.to_thread` inference、disconnect/explicit close 或 session registry teardown 变化。 +- 强制：昂贵的 deferred cognition 必须在 sole input reader 之外运行，完成 watcher 而不是 input + consumption 才能发起 response；response start/replacement 通过一个 response lock 串行化，新 response + 以 stale epoch 加 cancellation 取代 live response。`response.cancel` 无论有无 in-flight response 都必须 + barge-in 并发出 cancelled acknowledgement。不可取消的 CUDA worker 在 asyncio wrapper cancellation 后仍须 + 在 shield 下等待真实退出，且显式处理 `asyncio.CancelledError`，共享 inference lock 在 drain 前不得释放。 + deferred task 必须按 session 追踪；barge-in/disconnect cancel 并 drain，explicit graceful close 才调用 + `flush(session)`，随后 `on_close(session)` 释放 session state。shared lifecycle hook 必须对旧的 + duck-typed adapter 保持 optional/compatible；registry/lifespan close 必须 close runtime 并清空 lease。 +- 禁止：在 reader path await gate/model cognition；copy 一份 model-specific serving loop/runtime；把 wrapper + cancellation 当成 GPU worker 已停止；silent no-op `response.cancel`；遗留 gate task/queue/session map；或把 + Mage-specific flush/response-on-close policy 施加给 generic adapter。 +- 验收：two-second synthetic gate 下 `response.cancelled` 在 0.5 seconds 内到达；并发 input/gate completion + 不得创建 duplicate response；cancel/disconnect/close 后无 worker/task、registry lease 或 session-state residue； + explicit close 可 flush pending gate 再完成 response，disconnect 则 cancel；没有 lifecycle method 的既有 + adapter 行为不变。PR 的 CPU suite 证明 lifecycle contract；报告的 RTX 4090 run 只是 single-run functional + evidence，不得外推 throughput、latency 或一般 hardware support。^[PR #6537] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 7f8067f7c..2e1756096 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -48,7 +48,7 @@ confidence: high | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 见 engine lifecycle；`SERV-5s` 见 [app assembly](rules-app-assembly.md) | | `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup/shutdown race、streaming raw terminal、event-driven orchestration、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5r`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）、[SERV-5t](rules-abort-lifecycle.md#serv-5t-acknowledged-abort-在-shutdown-边界只忽略已识别的-transport-closure)，`SERV-5f` | -| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion 与 stage-keyed segment lifecycle | `SERV-6a`–`SERV-6f` 见 [engine 生命周期规则](rules-engine-lifecycle.md)；`SERV-6g` 见 [duplex 编排规则](rules-duplex-orchestration.md) | +| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion、stage-keyed segment lifecycle，或 adapter-owned deferred cognition/session cleanup | `SERV-6a`–`SERV-6f` 见 [engine 生命周期规则](rules-engine-lifecycle.md)；`SERV-6g` 见 [duplex 编排规则](rules-duplex-orchestration.md)；`SERV-6h` 见 [session lifecycle](rules-session-lifecycle.md) | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds、transform artifact ownership | `SERV-4a`–`4h`, `SERV-4l`–`4q`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d5fbdd2f3..59e899027 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b5e898f2`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 48ae412a`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 4927fd54 \ - --to b5e898f2 \ + --from b5e898f2 \ + --to 48ae412a \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/mage-vl/_index.md b/knowledge/repos/vllm-omni/models/mage-vl/_index.md new file mode 100644 index 000000000..170fc9f10 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/mage-vl/_index.md @@ -0,0 +1,35 @@ +--- +title: "Mage-VL" +created: 2026-09-05 +updated: 2026-09-05 +type: index +tags: [vllm-omni, models, model-executor] +sources: ["PR #6537", recipes/Microsoft/Mage-VL.md, examples/offline_inference/mage_vl/end2end.py, examples/online_serving/mage_vl/duplex_client.py, vllm_omni/experimental/fullduplex/mage_vl/] +confidence: high +--- + +# Mage-VL + +## 名称、范围与入口 + +- 实验性 checkpoint 是 `microsoft/Mage-VL`。代码 owner 是 + `vllm_omni/experimental/fullduplex/mage_vl/`，不是 `model_executor/models/`。 +- 离线入口是 `examples/offline_inference/mage_vl/end2end.py`；独立 server 的 public transport 是 + `WS /v1/mage-vl/duplex`，client 是 `examples/online_serving/mage_vl/duplex_client.py`。 +- checkpoint 由 Transformers remote code 加载：`AutoProcessor` 与 `AutoModelForCausalLM` 均使用 + `trust_remote_code=True`。共享 deferred-session lifecycle 见 + [Serving session lifecycle](../../components/serving/rules-session-lifecycle.md)。 + +## 明确不属于本 owner 的声明 + +PR #6537 没有 native vLLM Mage-VL architecture/`_OMNI_MODELS` registration、`OMNI_PIPELINES` key、 +deploy YAML 或 unified `/v1/duplex` route；也不能把这个 experimental standalone adapter 写入正常 +catalog/supported-model family。未来 native executor support 必须以独立变更和真实 checkpoint compatibility +evidence 证明。^[PR #6537] + +## 目录内容 + +| 遇到什么 | 查看哪里 | +|---|---| +| remote-code backend、encoded-video transport、codec/frames boundary、session/runtime topology | [architecture](architecture.md) | +| bounded state、gate、cancellation、capability 与 non-native boundary | [rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/mage-vl/architecture.md b/knowledge/repos/vllm-omni/models/mage-vl/architecture.md new file mode 100644 index 000000000..84d29a246 --- /dev/null +++ b/knowledge/repos/vllm-omni/models/mage-vl/architecture.md @@ -0,0 +1,34 @@ +--- +title: "Mage-VL experimental architecture" +created: 2026-09-05 +updated: 2026-09-05 +type: architecture +tags: [vllm-omni, models, model-executor] +sources: ["PR #6537", vllm_omni/experimental/fullduplex/mage_vl/adapter.py, vllm_omni/experimental/fullduplex/mage_vl/serving/backend.py, vllm_omni/experimental/fullduplex/mage_vl/serving/server.py, recipes/Microsoft/Mage-VL.md] +confidence: high +--- + +# Mage-VL experimental architecture + +## Transport → adapter → backend + +Client sends shared duplex `input.append`: optional text plus encoded MP4 video data (`video_base64` and +segment/timing metadata) to standalone `WS /v1/mage-vl/duplex`. The FastAPI server owns bearer authentication, +session-ID validation, concurrent-session/message bounds, idle timeout, cancellation and lease cleanup. It advertises +only `{text, video}` input and `{text}` output, matching the checked-in Transformers backend. + +`MageVLDuplexAdapter` owns per-session bounded causal visual windows, pending query/gate decision, event +de-duplication and gate tasks. `MageVLDuplexRuntime` is a small shared-runtime subclass that watches deferred gate +completion while the reader remains available for cancellation. The common response/cancellation/close mechanics belong +to [SERV-6h](../../components/serving/rules-session-lifecycle.md). + +`MageVLTransformersBackend` materializes the encoded MP4 temporarily, applies the remote-code processor, runs the +StreamMind gate or generation, and serializes access to the shared checkpoint. `frames` samples decoded frames. The +optional `codec` backend requires `ffmpeg`, `ffprobe`, and `cv-preinfer` from `codec-video-prep>=0.2.5`; absent +`cv-preinfer` is a startup error with `frames` as the fallback. + +## Boundary + +The adapter has generic codec-window abstractions, but the production Transformers transport accepts encoded video +only. Image or opaque `codec_window` must not be advertised as production capabilities. This is not a native vLLM +executor, pipeline, deploy profile, or unified route.^[PR #6537] diff --git a/knowledge/repos/vllm-omni/models/mage-vl/rules.md b/knowledge/repos/vllm-omni/models/mage-vl/rules.md new file mode 100644 index 000000000..75858498c --- /dev/null +++ b/knowledge/repos/vllm-omni/models/mage-vl/rules.md @@ -0,0 +1,32 @@ +--- +title: "Mage-VL 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, models, model-executor] +sources: ["PR #6537", recipes/Microsoft/Mage-VL.md, examples/offline_inference/mage_vl/end2end.py, examples/online_serving/mage_vl/duplex_client.py, vllm_omni/experimental/fullduplex/mage_vl/adapter.py, vllm_omni/experimental/fullduplex/mage_vl/serving/backend.py, vllm_omni/experimental/fullduplex/mage_vl/serving/server.py, tests/e2e/features/fullduplex/test_mage_vl_adapter.py, tests/e2e/features/fullduplex/test_mage_vl_serving.py, tests/examples/offline_inference/test_mage_vl.py] +confidence: high +--- + +# Mage-VL 规则 + +## MAGEVL-1a — experimental adapter 必须保持 remote-code、bounded duplex 与 non-native boundary + +- 触发：Mage-VL loader/backend、adapter capability、visual window/gate flow、WebSocket protocol、codec backend、 + documentation/example 或 attempted registry/deploy/unified-route change。 +- 强制：用 `AutoProcessor.from_pretrained(..., trust_remote_code=True)` 与 + `AutoModelForCausalLM.from_pretrained(..., trust_remote_code=True)` 加载 checkpoint，并在 serving 前验证 + StreamMind gate method。mutable window、pending query/gate decision、dedup ID 与 gate task 必须 per-session + isolation、bounded causal history，且拒绝 `window_size > max_windows`。production WebSocket 只接受 text 加 + encoded video，并使用 dedicated `/v1/mage-vl/duplex`；base64 video materialization 必须在全部 prepare + failure/cancellation path cleanup。`codec` 必须在 `cv-preinfer` 缺失时 fail fast 并说明 `frames` fallback； + backend GPU inference 与 cancellation/close 遵循 [SERV-6h](../../components/serving/rules-session-lifecycle.md)。 +- 禁止：把 Mage-VL 注册为 Qwen3-VL 或任意 native vLLM architecture、写入 global `ModelRegistry`、增加 + pipeline/deploy config，或宣称 unified route/ordinary `vllm serve` support；advertise 当前 Transformers + transport 不接受的 image/opaque `codec_window`；把 RTX 4090 functional run 说成 benchmark、general performance + 或 general hardware-support evidence。 +- 验收：CPU tests 覆盖 text+encoded-video、exact capability、bounded window、gate dedup、cross-session isolation、 + cancellation responsiveness、duplicate-session close code 4429、cleanup、codec missing-binary failure 与 frames + fallback。offline wrapper 必须验证 Mage checkout/media/dependencies，并构造 image、frame-video、codec-video 与 + streaming reference command。未来 native registration 必须单独证明 real checkpoint processor/config compatibility、 + real vLLM execution 以及 registry/pipeline/deploy/unified-route validation。^[PR #6537] From 367cff202bb51e6a5e2e9bd6ece375ae179fb314 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 07:59:30 +0800 Subject: [PATCH 441/465] knowledge(vllm-omni): sync 24c49cf3 (PR #5568) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 3 +- .../diffusion/rules-custom-op-dispatch.md | 33 +++++++++++++++++++ .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 10 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 knowledge/repos/vllm-omni/components/diffusion/rules-custom-op-dispatch.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 607a20bb2..4c75eaf2f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b5e898f295382b0d3457da95d0901dda3701ebb8 + previous_audited_sha: 48ae412a2f6feb3c097cbc915d6427a163e454e3 audited_ref: main - audited_sha: 48ae412a2f6feb3c097cbc915d6427a163e454e3 + audited_sha: 24c49cf392c7c087babbcb38f91a6b260b07c428 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 84b6cf129..18b247dd9 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -48ae412a2f6feb3c097cbc915d6427a163e454e3`. The canonical machine baseline +24c49cf392c7c087babbcb38f91a6b260b07c428`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 49c62f590..acc8eaeb3 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 48ae412a` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 24c49cf3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f1a9f7b0b..bf01f778c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 48ae412a` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 24c49cf3` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -34,6 +34,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ | vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | | worker process title、拓扑局部 rank 与日志前缀 | [worker observability rules](rules-worker-observability.md) | | 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | +| CustomOp XPU platform dispatch、PyTorch-native fallback、CUDA contract 与非同义 override 边界 | [CustomOp dispatch rules](rules-custom-op-dispatch.md) | | runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | | Wan VAE height/width spatial reshard、empty tail、attention extent | [Wan spatial-shard rules](rules-wan-spatial-shard.md) | | multi-DiT、dotted `_dit_modules`、loader-to-offloader handoff 与跨 Cache-DiT/compile/LoRA/offload lifecycle | [component lifecycle rules](rules-component-lifecycle.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-custom-op-dispatch.md b/knowledge/repos/vllm-omni/components/diffusion/rules-custom-op-dispatch.md new file mode 100644 index 000000000..a9b6510f6 --- /dev/null +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-custom-op-dispatch.md @@ -0,0 +1,33 @@ +--- +title: "Diffusion CustomOp dispatch 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, components, diffusion] +sources: ["PR #5568", vllm_omni/diffusion/layers/custom_op.py, tests/diffusion/layers/test_custom_op.py, vllm_omni/model_executor/models/common/qwen3_code_predictor.py] +confidence: high +--- + +# Diffusion CustomOp dispatch 规则 + +## DIFF-1ae — XPU CustomOp 默认分派必须回落 `forward_native`，但不得抹平非同义 backend 路径 + +- 触发：修改 `CustomOp` 的平台分派/默认 backend 方法，或为 shared diffusion `CustomOp` + 增删 XPU override。 +- 强制：`dispatch_forward()` 在 XPU 上仍绑定 `forward_xpu`；基类 + `forward_xpu(*args, **kwargs)` 必须直接委托 `forward_native(*args, **kwargs)`，以使没有 + 专用 XPU kernel 的 op 可走 PyTorch-native 路径。`forward_cuda` 仍须保持 + `NotImplementedError`，不得因 XPU fallback 而掩盖缺失 CUDA 实现。仅删除与 + `forward_native` 在参数和语义上完全等价的 subclass delegation override。 +- 禁止：把 XPU 当作自动直达 `forward_native` 而绕过 platform dispatch；将此 fallback 改为 + CUDA kernel fallback，或删除承担不同实现语义的 override。特别是 + `qwen3_code_predictor._RotaryEmbedding.forward_xpu` 必须保留：它选择缓存的 cos/sin + lookup，而 `forward_native` 进行 on-the-fly HF-reference math。`attention/backends/` + 的 `AttentionImpl.forward_xpu` 不属于本规则的清理范围。 +- 验收：CPU 上直接调用的 base-class 测试须断言 native-only op 的 `forward_xpu` 与 + `forward_native` 相等、`forward_cuda` 仍抛 `NotImplementedError`，并断言 + `MoTRMSNorm` 没有 subclass override 时可通过继承 fallback 处理带/不带 index 的调用。 + PR 报告的有界 XPU 证据为 4-card Intel XPU 上 BAGEL-7B-MoT（1024x1024、30 steps、 + layer-wise offload、TP=1）由失败变为 84s 完成，以及 Z-Image-Turbo（25 steps、CPU + offload、TP=1）82s 通过；这些运行不构成全部 XPU/custom-op 的兼容性清单或性能承诺。 + ^[PR #5568] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c329aafa7..cf9d5684b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 48ae412a` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 24c49cf3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 70301ab16..bbf387d51 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 48ae412a` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 24c49cf3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c6254897f..ee7ce0fcf 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 48ae412a` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 24c49cf3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index a878e42ab..c1cadf262 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 48ae412a` 验证存在 +- 源码校验：以上路径均已在 `main @ 24c49cf3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 59e899027..5fe057a58 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 48ae412a`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 24c49cf3`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from b5e898f2 \ - --to 48ae412a \ + --from 48ae412a \ + --to 24c49cf3 \ --repo \ --mode report-only ``` From b404f52955ebbbebf5d40c7f6a3049616624796f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:02:01 +0800 Subject: [PATCH 442/465] knowledge(vllm-omni): sync d1a5a7c1 (PR #6886) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/qwen-omni/_index.md | 7 +++++-- .../repos/vllm-omni/models/qwen-omni/architecture.md | 9 ++++++++- knowledge/repos/vllm-omni/models/qwen-omni/rules.md | 9 ++++++++- 12 files changed, 33 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4c75eaf2f..a0c7c2c4f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 48ae412a2f6feb3c097cbc915d6427a163e454e3 + previous_audited_sha: 24c49cf392c7c087babbcb38f91a6b260b07c428 audited_ref: main - audited_sha: 24c49cf392c7c087babbcb38f91a6b260b07c428 + audited_sha: d1a5a7c178d0032eccb836eb5cbee16548484d1f # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 18b247dd9..171629eca 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -24c49cf392c7c087babbcb38f91a6b260b07c428`. The canonical machine baseline +d1a5a7c178d0032eccb836eb5cbee16548484d1f`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index acc8eaeb3..01f6c8c7a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 24c49cf3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ d1a5a7c1` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index bf01f778c..2b6a4fa86 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 24c49cf3` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ d1a5a7c1` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index cf9d5684b..0c5de5085 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 24c49cf3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ d1a5a7c1` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index bbf387d51..e78cd3efe 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 24c49cf3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ d1a5a7c1` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ee7ce0fcf..ac90f3dfd 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 24c49cf3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ d1a5a7c1` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index c1cadf262..bc7977b7a 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 24c49cf3` 验证存在 +- 源码校验：以上路径均已在 `main @ d1a5a7c1` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5fe057a58..1f7500c0e 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 24c49cf3`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ d1a5a7c1`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 48ae412a \ - --to 24c49cf3 \ + --from 24c49cf3 \ + --to d1a5a7c1 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index 4b9a8b3bf..d5bc16976 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-05 type: index tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/quantization/component_config.py, docs/design/qwen3_omni_tts_performance_optimization.md] +sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322", "PR #6886", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/quantization/component_config.py, docs/design/qwen3_omni_tts_performance_optimization.md] --- # Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS） @@ -34,6 +34,9 @@ sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322" `vllm serve` 缺 `model_stage` 时 Qwen2.5/Qwen3 默认 thinker，Qwen3 thinker 在非 staged 模式返回 bare tensor，只有 staged talker consumer 才请求 captured layers；共享验收见 [EXEC-1e](../../components/model-executor/rules-bridge-batch.md#exec-1e-upstream-registry-重名时-omni-override-与-plain-vllm-forward-必须同时成立)。^[PR #5976] +- Qwen2.5 的 code2wav/Token2Wav 缺失、speaker resource loading 或旧 direct speech helper + 问题，必须按 staged Thinker → Talker → Code2Wav ownership 处理；soft-fail、有效 HF-folder + initializer caller 与已删除 helper 的边界见本目录 Qwen-Omni rules。^[PR #6886] - 官方历史文档：`docs/design/module/archive/async_omni_architecture.md`（以 Qwen3-Omni 为 worked example 的分层运行时快照，非 active spec）、 `docs/design/qwen3_omni_tts_performance_optimization.md`（性能优化实录） @@ -53,4 +56,4 @@ sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322" | 遇到什么 | 查看哪里 | |---|---| | stage 拓扑、代际差异与官方性能优化结论 | [architecture](architecture.md) | -| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary、AWQ/compressed-tensors 名称映射、固定种子音频回归，或 audio-encoder head/TP divisibility | [Qwen-Omni rules](rules.md) | +| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary、AWQ/compressed-tensors 名称映射、固定种子音频回归、audio-encoder head/TP divisibility，或 Qwen2.5 code2wav soft-fail/旧 speech helper | [Qwen-Omni rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md index 137f65952..f57f155e6 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-04 type: architecture tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", "PR #5671", "PR #6284", docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/config/model.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/registry.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/worker/gpu_model_runner.py] +sources: ["PR #5073", "PR #5671", "PR #6284", "PR #6886", docs/design/qwen3_omni_tts_performance_optimization.md, docs/design/module/engine_orchestration.md, docs/design/module/stage_runtime.md, docs/design/module/archive/async_omni_architecture.md, vllm_omni/config/model.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/registry.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/worker/gpu_model_runner.py] --- # Qwen-Omni 家族拓扑与性能结论 @@ -22,6 +22,13 @@ sources: ["PR #5073", "PR #5671", "PR #6284", docs/design/qwen3_omni_tts_perform `qwen2_5_omni_thinker_only` pipeline 提供纯理解形态——hunyuan 架构页把 `qwen2_5_omni_thinker.py` 标为 **I2T blessed pattern**（新模型 I2T 适配的照抄 基准）。 +- Qwen2.5-Omni 同样按 stage 分别构造：Thinker 产出文本 hidden states，Talker 产出 + codec token，Code2Wav 独占 Token2Wav 并将 codec 解码成波形。PR #6886 因此删除未被 + staged pipeline 使用的 `generate_speech` 及其私有 converter，而非只补 sampling metadata： + 后者仍会以额外参数调用仅接受 `hidden_states` 的 Talker `compute_logits`，且没有已构造 + instance 同时拥有 Talker 与 Token2Wav。Code2Wav 缺 Token2Wav 时 `_codec_to_audio` 返回 + `None`；speaker resource initializer 只由有 HF folder 的 weight-loading path 调用。该 + deletion-only PR 没有新增测试、E2E、质量或性能证据。^[PR #6886] - Qwen3-Omni 的 pipeline 由 resolver（`models/qwen3_omni/pipeline.py:: resolve_qwen3_omni_pipeline`）按 checkpoint 结构动态决定，而不是冻结的 `PipelineConfig` 字面量。 diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index fe7702bc7..2488d2b38 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py] +sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", "PR #6886", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py] confidence: high --- @@ -58,3 +58,10 @@ confidence: high - 强制：以 `encoder_attention_heads % global_tp_size` 判定该 audio-encoder layer 是否可分片。可整除时保持 global TP；不可整除时仅该 encoder layer 的 attention QKV、attention output projection、FC1 和 FC2 都以 `disable_tp=True` 构造，且 local attention head count 按 effective TP=1 计算。 - 禁止：仅关闭 QKV 或仅修正 `num_local_heads` 而让 output/FFN 仍分片；把这项 fallback 扩大为整个 Qwen3-Omni 模型关闭 TP；或把 PR 的 TP1/2/4/8 手工音频请求、MI350x full-workload throughput 数字归因于该 fallback。 - 验收：分别覆盖 heads 可被和不可被 global TP 整除的构造，断言四个 linear 的 `disable_tp` 一致、attention local-head count 正确，且 encoder 外的 Thinker/Talker TP 配置未被此条件改写。PR 仅报告手工请求与整体 workload 测量，未给出该局部回退的独立性能或质量证明。^[PR #4322] + +## QOMNI-1e — Qwen2.5-Omni code2wav 缺失时必须 soft-fail，不能复活跨 stage 的旧 speech helper + +- 触发：修改 Qwen2.5-Omni 的 `generate_audio`、`_codec_to_audio`、`_init_token2wav_model`、weight loading、Talker sampling/logits，或 Thinker/Talker/Code2Wav 的 staged 构造。 +- 强制：`_codec_to_audio` 在 `token2wav` 缺失时必须直接返回 `None`；`_init_token2wav_model` 只能由持有已解析 `hf_model_folder` 的 Token2Wav weight-loading 路径调用；语音生成维持 Thinker → Talker → Code2Wav 的 stage handoff。 +- 禁止：无参数调用 `_init_token2wav_model()`；恢复 `generate_speech` 或 `_convert_to_codec_tokens`；仅 thread `sampling_metadata` 就声称修复旧 helper（其 Talker `compute_logits` 调用仍带已失效的额外参数）；或让单一 wrapper 违反 staged ownership 而同时拥有 Talker 和 Token2Wav。 +- 验收：目标树中不存在两个已删除 helper 或无参数 initializer call；`token2wav is None` 的 `_codec_to_audio` 路径返回 `None` 而非 `TypeError`；Token2Wav load path 仍向 initializer 传入解析后的 HF directory。PR 未新增测试或 E2E，故这些是静态/call-site 合同，不构成运行时、音频质量或性能证据。^[PR #6886] From b1606d032f73a4e2930f15683b150f122ea8e61a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:04:11 +0800 Subject: [PATCH 443/465] knowledge(vllm-omni): sync 2b47faf6 (PR #6230) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../configuration/rules-topology-profiles.md | 23 ++++++++++++++++++- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 10 files changed, 34 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index a0c7c2c4f..c88ce0195 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 24c49cf392c7c087babbcb38f91a6b260b07c428 + previous_audited_sha: d1a5a7c178d0032eccb836eb5cbee16548484d1f audited_ref: main - audited_sha: d1a5a7c178d0032eccb836eb5cbee16548484d1f + audited_sha: 2b47faf6012ad3e9c4c2e6668b313a3aea85066b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 171629eca..681b7e3ab 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -d1a5a7c178d0032eccb836eb5cbee16548484d1f`. The canonical machine baseline +2b47faf6012ad3e9c4c2e6668b313a3aea85066b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 01f6c8c7a..c6b9379c8 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ d1a5a7c1` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 2b47faf6` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md index 4b5299fd3..38562dc19 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules-topology-profiles.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, components, config] -sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_runtime.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/deploy/, "PR #6186", "PR #6813", "PR #6291", "PR #6829", vllm_omni/config/pipeline_registry.py, tests/config/test_config_factory.py, tests/config/test_omni_config.py, tests/config/test_pipeline_registry.py, tests/utils/test_tracking_parser.py] +sources: ["PR #4222", "PR #4795", "PR #5604", "PR #5842", "PR #5885", "PR #6082", vllm_omni/config/stage_config.py, vllm_omni/config/config_factory.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_runtime.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/deploy/, "PR #6186", "PR #6813", "PR #6291", "PR #6829", vllm_omni/config/pipeline_registry.py, tests/config/test_config_factory.py, tests/config/test_omni_config.py, tests/config/test_pipeline_registry.py, tests/utils/test_tracking_parser.py, "PR #6230", vllm_omni/entrypoints/utils.py, tests/entrypoints/test_serve.py, tests/entrypoints/test_utils.py] confidence: high --- @@ -96,3 +96,24 @@ confidence: high `final_output_type` 推断 terminal；不得声称 resolver 后有独立 revalidation、cycle/runtime/E2E proof。 - 验收：无 stages/terminal、duplicate ID、invalid/self input eager error；terminal-not-last 与有效 fixture pass；static configs pass，resolver concrete result materialization guarded。^[PR #6291] ^[PR #6829] + +## CONF-5k — `--stage-overrides` parser 只验证 shape，字段语义留给下游 owner + +- 触发：修改 `parse_stage_overrides`、`--stage-overrides` CLI 输入、stage-scoped + override forwarding，或考虑在 parser 对字段名、值类型、范围或 stage existence 做校验。 +- 强制：对 falsey raw input（包括 `None`、`False`、`0`、空 string/容器）在任何验证前返回 + `None`；truthy string 先 JSON parse，再要求 top-level 为 dict、每个 key 为 ASCII + `isdigit()` string、每个 value 为 dict。非空 outer dict 中的空 inner dict 合法；parse 后的 + 空 dict 返回 `None`。保留所有 inner fields 原样，不在 parser 建 allowlist，也不在此层做 + field type/range 或 stage-id existence 校验。 +- 禁止：把 `[]`、`0`、`false` 等 truthy JSON string 的 parsed scalar/container 当作 + “无 override”接受；用只含 deploy schema 的 allowlist 拒绝 `extras` 或 + `kv_cache_dtype`、`seed`、`swap_space`、`block_size`、`stage_connector_spec` 等 + stage-scoped engine args；用 `str.isdigit()` 而不加 ASCII guard，导致 fullwidth digit + 绕过并在下游错误路由。 +- 验收：覆盖 falsey raw passthrough、invalid JSON、truthy JSON list/scalar/bool rejection、 + non-ASCII/non-string/negative/float stage key rejection、non-dict inner value rejection， + 以及 `{"0": {}, "1": {}}` pass-through。再证明 `extras` 到 default-diffusion fallback/ + registered stage merge 的既有 owner、surviving `stage__` 到 `cli_overrides` 的 + forwarding 均未被 parser 截断；未知 `OmniEngineArgs` field 仅在 + `filter_dataclass_kwargs` 被 drop 并留下命名该 field 的 WARNING。^[PR #6230] diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2b6a4fa86..2c8bf7581 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ d1a5a7c1` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 2b47faf6` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 0c5de5085..53dedb204 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ d1a5a7c1` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 2b47faf6` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e78cd3efe..22eb9bbe1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ d1a5a7c1` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 2b47faf6` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ac90f3dfd..48ca30b4f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ d1a5a7c1` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 2b47faf6` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index bc7977b7a..694d04102 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ d1a5a7c1` 验证存在 +- 源码校验：以上路径均已在 `main @ 2b47faf6` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1f7500c0e..6c04033ab 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ d1a5a7c1`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 2b47faf6`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 24c49cf3 \ - --to d1a5a7c1 \ + --from d1a5a7c1 \ + --to 2b47faf6 \ --repo \ --mode report-only ``` From 3e92e5f4f3ecf8dc419508289ec95f412465299d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:06:31 +0800 Subject: [PATCH 444/465] knowledge(vllm-omni): sync 711daaca (PR #6955) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- .../vllm-omni/models/cosyvoice3/_index.md | 6 +++--- .../models/cosyvoice3/architecture.md | 8 ++++++-- .../repos/vllm-omni/models/cosyvoice3/rules.md | 18 ++++++++++++++++-- 12 files changed, 37 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c88ce0195..8e1bf6865 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: d1a5a7c178d0032eccb836eb5cbee16548484d1f + previous_audited_sha: 2b47faf6012ad3e9c4c2e6668b313a3aea85066b audited_ref: main - audited_sha: 2b47faf6012ad3e9c4c2e6668b313a3aea85066b + audited_sha: 711daacabf1b4aecbb1492623a5d4df84b158656 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 681b7e3ab..24801ddd2 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -2b47faf6012ad3e9c4c2e6668b313a3aea85066b`. The canonical machine baseline +711daacabf1b4aecbb1492623a5d4df84b158656`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index c6b9379c8..9f605df2b 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 2b47faf6` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 711daaca` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 2c8bf7581..c6fa2eb29 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 2b47faf6` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 711daaca` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 53dedb204..e020e27e6 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 2b47faf6` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 711daaca` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 22eb9bbe1..792851826 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 2b47faf6` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 711daaca` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 48ca30b4f..ca72be5ac 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 2b47faf6` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 711daaca` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 694d04102..aa5240dc3 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 2b47faf6` 验证存在 +- 源码校验：以上路径均已在 `main @ 711daaca` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6c04033ab..5199ec63d 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 2b47faf6`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 711daaca`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from d1a5a7c1 \ - --to 2b47faf6 \ + --from 2b47faf6 \ + --to 711daaca \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md b/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md index 44a5e60d7..71a29853b 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/_index.md @@ -1,10 +1,10 @@ --- title: "CosyVoice3（单架构双 stage,RAS 合并停止,TRT 加速）" created: 2026-07-21 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: ["PR #5673", "PR #5869", "PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoice3.yaml, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py] +sources: ["PR #5673", "PR #5869", "PR #6424", "PR #6955", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/, vllm_omni/deploy/cosyvoice3.yaml, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py] --- # CosyVoice3 @@ -51,7 +51,7 @@ sources: ["PR #5673", "PR #5869", "PR #6424", vllm_omni/data_entry_keys.py, vllm | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | RAS 停止机制、双交接注册、TRT 门 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | -| CosyVoice3 TensorRT CFM 的 stream handoff、allocator lifetime 或 context pool | [rules](rules.md) | `COSYVOICE3-1a` 的顺序、所有权与验收 | +| CosyVoice3 TensorRT CFM 的 stream handoff、allocator lifetime、context pool、plan 并发发布、tmp 所有权或 cleanup failure | [rules](rules.md) | `COSYVOICE3-1a/1b` 的顺序、发布边界与验收 | ## 配置与 checkpoint 差异 diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md b/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md index 583de87a3..3997cf779 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/architecture.md @@ -1,10 +1,10 @@ --- title: "CosyVoice3 架构" created: 2026-07-21 -updated: 2026-09-04 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models] -sources: ["PR #5673", "PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py, vllm_omni/model_executor/models/cosyvoice3/pipeline.py, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py, vllm_omni/deploy/cosyvoice3.yaml] +sources: ["PR #5673", "PR #6424", "PR #6955", vllm_omni/data_entry_keys.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py, vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py, vllm_omni/model_executor/models/cosyvoice3/pipeline.py, vllm_omni/model_executor/stage_input_processors/cosyvoice3.py, vllm_omni/transformers_utils/configs/cosyvoice3.py, vllm_omni/deploy/cosyvoice3.yaml] --- # CosyVoice3 架构 @@ -23,6 +23,10 @@ sources: ["PR #5673", "PR #6424", vllm_omni/data_entry_keys.py, vllm_omni/model_ - **TRT 加速（本清单独有）**：campplus 说话人嵌入与 CFM DiT 估计器都可走 TensorRT（启动时 ONNX→plan,按设备缓存 plan;campplus 有 CPU-ONNX 兜底）, 统一由 `COSYVOICE3_TRT` 门控,默认开。 +- **CFM plan 发布**：flow-estimator builder 先序列化 engine bytes，再经自有、同目录的独占 + 临时 plan 写入并 close，最后以 `os.replace` 发布至缓存路径；因此并发 builder 可各自完成 + artifact，最终缓存允许 last-writer-wins。失败清理仅处理 writer 自己创建的 tmp，不承担缓存 + 协调或持久化职责；具体改动约束见 [rules](rules.md)。^[PR #6955] - **CFM TensorRT handoff**：每个 execution context 与一个原始 `torch.cuda.Stream` 成对入池；caller stream 先由该 stream 的 `wait_stream` 建立 producer→TensorRT 依赖，TRT 在该 stream 的 raw `cuda_stream` handle 上 diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md b/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md index f34b4c6d7..3aa8086c1 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md @@ -1,10 +1,10 @@ --- title: "CosyVoice3 规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models] -sources: ["PR #5673", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py, vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py, tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py, benchmarks/tts/benchmark_cosyvoice3_trt_streams.py] +sources: ["PR #5673", "PR #6955", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py, vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py, tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py, benchmarks/tts/benchmark_cosyvoice3_trt_streams.py] confidence: high --- @@ -30,3 +30,17 @@ confidence: high input/output 的 allocator-lifetime 与数值 parity。性能报告要分开写 submission、engine 和 E2E 指标：受控 handoff microbenchmark 只能证明消除了 host blocking，有限的 E2E 样本或 engine 小幅变化不能声称模型吞吐/延迟提升。^[PR #5673] + +## COSYVOICE3-1b — flow-estimator TRT plan 必须以自有临时文件原子发布 + +- 触发：修改 CosyVoice3 `flow_estimator_trt.py` 的 ONNX→plan 序列化、plan 缓存路径或 + `_write_plan_atomically` 发布流程。 +- 强制：在 destination 同目录以 `.tmp..` 独占 `xb` 创建临时文件；写完并 + close 后才 `os.replace` 到 plan。并发成功者各自发布完整 artifact，最终允许 last-writer-wins。 + write 或 replace 失败时只清理本调用已创建的 tmp；cleanup 失败仅记录日志，必须重抛原始异常。 +- 禁止：使用固定共享 `.tmp` 名称；在独占创建 collision 时删除 foreign tmp；把此发布合同扩展为 + `fsync`、目录持久化、锁、内容校验或事务保证，或把 CPU 文件系统测试表述为 TensorRT/GPU 行为证明。 +- 验收：`tests/model_executor/models/cosyvoice3/test_flow_estimator_trt.py` 的五项 CPU 覆盖必须分别 + 证明 replace 失败保留旧 plan、cleanup 失败保留 replace error、partial write 清理 owned tmp、 + collision 保留 foreign tmp，以及两条同步 publisher 使用不同 source path 且仅留下完整 final plan。 + ^[PR #6955] From bc75000df2ae61db06724e280bd3cee3f5011ef9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:08:27 +0800 Subject: [PATCH 445/465] knowledge(vllm-omni): sync ec22c7ac (PR #6723) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- .../components/serving/rules-app-assembly.md | 12 +++++++----- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 19 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8e1bf6865..2384b7e39 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 2b47faf6012ad3e9c4c2e6668b313a3aea85066b + previous_audited_sha: 711daacabf1b4aecbb1492623a5d4df84b158656 audited_ref: main - audited_sha: 711daacabf1b4aecbb1492623a5d4df84b158656 + audited_sha: ec22c7ac06ad1d344dbfcf6aaf8b641113bf87a5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 24801ddd2..c0f014201 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -711daacabf1b4aecbb1492623a5d4df84b158656`. The canonical machine baseline +ec22c7ac06ad1d344dbfcf6aaf8b641113bf87a5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 9f605df2b..bd6383024 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 711daaca` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ec22c7ac` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index c6fa2eb29..7395d65be 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 711daaca` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ ec22c7ac` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index e020e27e6..19c6b8f2f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 711daaca` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ec22c7ac` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 792851826..0a6bbd9cf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 711daaca` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ec22c7ac` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ca72be5ac..2e3b29b6f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 711daaca` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ec22c7ac` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index aa5240dc3..2fcc34c06 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 711daaca` 验证存在 +- 源码校验：以上路径均已在 `main @ ec22c7ac` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md b/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md index fd88f8d5d..db8234d48 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-app-assembly.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #6609", "PR #6707", vllm_omni/config/endpoint_policy.py, vllm_omni/entrypoints/openai/api_server.py, tests/config/test_endpoint_policy.py, tests/entrypoints/openai/test_profiler_endpoints.py, tests/entrypoints/openai_api/test_api_server_guards.py] +sources: ["PR #6609", "PR #6707", "PR #6723", vllm_omni/config/endpoint_policy.py, vllm_omni/entrypoints/openai/api_server.py, tests/config/test_endpoint_policy.py, tests/entrypoints/openai/test_profiler_endpoints.py, tests/entrypoints/openai_api/test_api_server_guards.py] confidence: high --- @@ -28,7 +28,9 @@ confidence: high client 收到原值。`/start_profile` 与 `/stop_profile` 是当前实例。PR #6707 现有测试没有直接 method partition 或 API assembly 覆盖；review 的批准边界是 static review、merge tree 与 CI，不能据此作 broad compatibility/E2E 声明。^[PR #6609] ^[PR #6707, merged 2026-09-02] -- 已知边界：同一 assembled app 的 upstream `GET /health` 仍先于 Omni health route 注册，当前 - first-match dispatch 会遮蔽 Omni 的 diffusion fallback/JSON handler；测试 fake upstream 也未注册 - 该 sibling route。该 review thread 在本提交获批时明确 defer，不能据 profiler 修复声称所有重复 - route 已清零，后续修复还需核对 render-only `engine_client is None` 时 200/503 的语义差异。^[PR #6609] +- 已修复边界：`build_openai_app()` 若先注册 upstream `GET /health`，必须在 + `app.include_router(router)` 前用 `remove_route_from_app(app, "/health", {"GET"})` 从 assembled app + 删除它；router-local 的预先删除发生在本地 health route 注册前，是 no-op。最终 assembled app 只保留 + 一个 `GET /health`，并由 Omni `health` handler 所有；这不改变 native vLLM 的 render-only health + route。现有证据仅为 CPU assembly guard：fake upstream health 后以 route count 和 endpoint identity + 锁定所有权，不构成 server E2E 或 render-only 200/503 语义验证。^[PR #6723, merged 2026-09-03] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5199ec63d..86e9a94ee 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 711daaca`（2026-09-02）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ec22c7ac`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 2b47faf6 \ - --to 711daaca \ + --from 711daaca \ + --to ec22c7ac \ --repo \ --mode report-only ``` From f321ed373b914d27e9f7fb0acaffb9a0f30e33d6 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:10:53 +0800 Subject: [PATCH 446/465] knowledge(vllm-omni): sync 1dae5373 (PR #6847) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../diffusion/rules-output-lifecycle.md | 9 ++++++--- .../vllm-omni/components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/ltx2/_index.md | 14 ++++++++------ .../repos/vllm-omni/models/ltx2/architecture.md | 17 +++++++++++------ 12 files changed, 37 insertions(+), 27 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 2384b7e39..34e8a48ec 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 711daacabf1b4aecbb1492623a5d4df84b158656 + previous_audited_sha: ec22c7ac06ad1d344dbfcf6aaf8b641113bf87a5 audited_ref: main - audited_sha: ec22c7ac06ad1d344dbfcf6aaf8b641113bf87a5 + audited_sha: 1dae53731389cce1d2f99697c920a4c159196548 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c0f014201..ff3675f6c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ec22c7ac06ad1d344dbfcf6aaf8b641113bf87a5`. The canonical machine baseline +1dae53731389cce1d2f99697c920a4c159196548`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bd6383024..2918a5f8f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ec22c7ac` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1dae5373` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7395d65be..8da63daaa 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ec22c7ac` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 1dae5373` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 421f47980..7520b1b39 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", "PR #6953", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] +sources: ["PR #5550", "PR #5864", "PR #5885", "PR #5978", "PR #6750", vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/inline_stage_diffusion_client.py, vllm_omni/diffusion/io_support.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/output_formatter.py, vllm_omni/diffusion/ipc.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/stage_diffusion_proc.py, vllm_omni/diffusion/utils/media_utils.py, vllm_omni/diffusion/worker/diffusion_worker.py, tests/diffusion/test_async_output_timeout.py, tests/diffusion/test_async_output_worker.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_engine_cleanup.py, tests/diffusion/test_diffusion_ipc.py, tests/diffusion/test_inline_stage_diffusion_client.py, tests/diffusion/test_ipc_async.py, tests/diffusion/test_multiproc_engine_concurrency.py, tests/diffusion/test_result_pump.py, tests/diffusion/test_stage_diffusion_proc.py, tests/entrypoints/openai_api/test_video_server.py, "PR #6023", "PR #5983", "PR #4222", "PR #6094", "PR #6255", "PR #6288", "PR #6308", "PR #6499", "PR #6749", "PR #6847", "PR #6953", vllm_omni/diffusion/data.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py, tests/diffusion/test_diffusion_output_formatter.py, tests/diffusion/test_diffusion_plugin_hooks.py, tests/entrypoints/openai_api/test_video_pipeline_capability.py] confidence: high --- @@ -137,9 +137,12 @@ confidence: high 可跨 orchestrator multiprocess 边界解析的 module-level post-process callable。SANA 的 `SanaVideoPipeline` 与 `SanaImageToVideoPipeline` 都必须 direct metadata 声明 `final_output_type="video"`；这是 serving metadata 分类，非 I2V input schema、denoise、topology - 或 VAE 合同变更。 + 或 VAE 合同变更。LTX 的 `LTX2DistilledOneStagePipeline` 与显式 + `LTX2DistilledTwoStagePipeline` 必须都经 metadata alias 收敛到 canonical + `LTX2DistilledPipeline` 的 video metadata；同一结论还必须在 registry + architecture→pipeline class 解析路径成立。 - 禁止：只在 topology 标 video 但 formatter 仍发 image；要求每个 stage 都是 diffusion 才允许 video final；遗漏已注册 video alias 后静默回 image；用 ndarray 形状猜语义，或把局部 closure 作为跨进程 hook。 -- 验收：逐个 canonical class/alias 断言 default 与 multi-stage final type，video/image endpoint 正负 capability 及最终 output type；action 另验证 post-process callable 可跨目标进程边界使用，最终 output 保留 stable key、shape、dtype 和有限性。PR #6953 的 CPU metadata unit test 只断言两种 SANA class 均分类为 video；PR 所述 online/offline 行为不是新的 endpoint 或真实模型 E2E 证据。^[PR #4222] ^[PR #5885] ^[PR #6953] +- 验收：逐个 canonical class/alias 断言 default 与 multi-stage final type，video/image endpoint 正负 capability 及最终 output type；action 另验证 post-process callable 可跨目标进程边界使用，最终 output 保留 stable key、shape、dtype 和有限性。PR #6953 的 CPU metadata unit test 只断言两种 SANA class 均分类为 video；PR #6847 的同类 CPU metadata test 只覆盖显式 LTX distilled two-stage alias 分类。两者都不是新的 endpoint、真实模型或 online/offline E2E 证据。^[PR #4222] ^[PR #5885] ^[PR #6847] ^[PR #6953] ## DIFF-9a — 共享 PyAV 预构造帧 mux 必须闭合资源与音频时间零点 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 19c6b8f2f..83e2ba227 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ec22c7ac` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1dae5373` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 0a6bbd9cf..e0abdae48 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ec22c7ac` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1dae5373` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 2e3b29b6f..e4511a81d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ec22c7ac` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1dae5373` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2fcc34c06..69dc93820 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ec22c7ac` 验证存在 +- 源码校验：以上路径均已在 `main @ 1dae5373` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 86e9a94ee..4fd37d5a5 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ec22c7ac`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1dae5373`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 711daaca \ - --to ec22c7ac \ + --from ec22c7ac \ + --to 1dae5373 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/ltx2/_index.md b/knowledge/repos/vllm-omni/models/ltx2/_index.md index d14511e1c..5980435df 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/_index.md +++ b/knowledge/repos/vllm-omni/models/ltx2/_index.md @@ -1,10 +1,10 @@ --- title: "LTX-2 家族（含 LTX-2.3/2.5）" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, ltx2] -sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, recipes/LTX/LTX-2.md, recipes/LTX/LTX-2.5.md] +sources: ["PR #6847", vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, recipes/LTX/LTX-2.md, recipes/LTX/LTX-2.5.md] --- # LTX-2 家族（含 LTX-2.3/2.5） @@ -14,10 +14,12 @@ sources: [vllm_omni/diffusion/models/ltx2/, vllm_omni/diffusion/registry.py, rec - 厂商/模型：Lightricks；22B 参数文本→视频+音频生成（T2V/I2V，48kHz 同步音频， 768x512 可达 20+ 秒）；Diffusers 格式 checkpoint `dg845/LTX-2.3-Diffusers` - 源码：`vllm_omni/diffusion/models/ltx2/`（纯 diffusion，无 AR stage） -- registry：`LTX2Pipeline` 统一 LTX-2/LTX-2.3 的 one-stage T2V/I2V； - `LTX2DistilledPipeline` 统一 distilled two-stage T2V/I2V；DMD2 仍由 - `LTX2T2VDMD2Pipeline`/`LTX2I2VDMD2Pipeline` 提供。旧的 LTX23、ImageToVideo 和 - TwoStages registry names 已删除，没有兼容 alias。 +- registry：`LTX2Pipeline`/`LTX2TwoStagePipeline` 分别提供 ordinary one-/two-stage， + `LTX2DistilledOneStagePipeline`/`LTX2DistilledTwoStagePipeline` 分别提供 distilled + one-/two-stage T2V/I2V；后两个 explicit class 都解析到 canonical + `LTX2DistilledPipeline` video metadata，后者同时保留为 distilled two-stage 的 legacy + alias。DMD2 仍由 `LTX2T2VDMD2Pipeline`/`LTX2I2VDMD2Pipeline` 提供。旧的 LTX23、 + ImageToVideo 和 plural `LTX2TwoStagesPipeline` registry names 已删除，没有兼容 alias。 - 官方 recipe 已合并为 `recipes/LTX/LTX-2.md`；已删除的 `recipes/LTX/LTX-2.3.md` 不能继续作为 source 或事实依据。 - 依赖共享 [Diffusion 组件](../../components/diffusion/_index.md) diff --git a/knowledge/repos/vllm-omni/models/ltx2/architecture.md b/knowledge/repos/vllm-omni/models/ltx2/architecture.md index 1a5448380..a6c348b40 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/architecture.md +++ b/knowledge/repos/vllm-omni/models/ltx2/architecture.md @@ -1,10 +1,10 @@ --- title: "LTX-2/2.3/2.5 模型架构与证据索引" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models, ltx2] -sources: [recipes/LTX/LTX-2.md, recipes/LTX/LTX-2.5.md, vllm_omni/diffusion/registry.py, "#4381", "#4464", "PR #6189", "PR #6342", vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_vae.py, tests/diffusion/models/ltx2/test_ltx2_vocoder_cuda.py, tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py, tests/e2e/accuracy/ltx/test_ltx_official_similarity.py] +sources: [recipes/LTX/LTX-2.md, recipes/LTX/LTX-2.5.md, vllm_omni/diffusion/registry.py, "#4381", "#4464", "PR #6189", "PR #6342", "PR #6847", vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_vae.py, tests/diffusion/models/ltx2/test_ltx2_vocoder_cuda.py, tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py, tests/e2e/accuracy/ltx/test_ltx_official_similarity.py] --- # LTX-2/2.3/2.5 模型架构与证据索引 @@ -19,10 +19,15 @@ sources: [recipes/LTX/LTX-2.md, recipes/LTX/LTX-2.5.md, vllm_omni/diffusion/regi - serving 入口（LTX-2.3）： `vllm serve diffusers/LTX-2.3-Diffusers --omni --stage-init-timeout 600`；统一 `LTX2Pipeline` 由 checkpoint metadata 选择 LTX-2 或 LTX-2.3 profile。 -- pipeline 变体：one-stage 使用 `LTX2Pipeline`，distilled two-stage 使用 - `LTX2DistilledPipeline`，DMD2 使用 `LTX2T2VDMD2Pipeline`/ - `LTX2I2VDMD2Pipeline`；T2V/I2V 通过是否提供 `image=` 选择，不再使用单独的 - `*ImageToVideoPipeline` registry names。 +- pipeline 变体：ordinary one-/two-stage 分别使用 `LTX2Pipeline`/ + `LTX2TwoStagePipeline`；distilled one-/two-stage 分别使用 + `LTX2DistilledOneStagePipeline`/`LTX2DistilledTwoStagePipeline`，其中 legacy + `LTX2DistilledPipeline` 是后者 alias。两个 explicit distilled class 的 serving metadata + 都解析到 canonical `LTX2DistilledPipeline` video metadata，避免 final stage 静默回落为 + image；DMD2 使用 `LTX2T2VDMD2Pipeline`/`LTX2I2VDMD2Pipeline`。T2V/I2V 通过是否提供 + `image=` 选择，不再使用单独的 `*ImageToVideoPipeline` registry names。PR #6847 仅以 CPU + metadata capability test 覆盖 explicit two-stage 的 video 分类，不证明 `/v1/videos`、真实模型 + load、recipe 执行或 output formatter 的 E2E 行为。 - LTX-2.5 的 video decode 将 canonical Native checkpoint 中的 decoder tensors 转换到 Diffusers-compatible DiffVAE；full 与 distilled 的 one-/two-stage profile 默认将其作为额外 video decoder component。ConvVAE 保留给 I2V encoding，也可由 startup-only From da3d5dd9845c893d69e0ec0ddec09e2a53cf7e8b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:13:18 +0800 Subject: [PATCH 447/465] knowledge(vllm-omni): sync dff28e68 (PR #6097) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/docs/repository-skills.md | 52 ++++++++++++++++--- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 10 files changed, 57 insertions(+), 19 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 34e8a48ec..d41867d97 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ec22c7ac06ad1d344dbfcf6aaf8b641113bf87a5 + previous_audited_sha: 1dae53731389cce1d2f99697c920a4c159196548 audited_ref: main - audited_sha: 1dae53731389cce1d2f99697c920a4c159196548 + audited_sha: dff28e686711add0e929ce0c5660b09ef26c8c88 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index ff3675f6c..d5ab9fa4f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -1dae53731389cce1d2f99697c920a4c159196548`. The canonical machine baseline +dff28e686711add0e929ce0c5660b09ef26c8c88`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2918a5f8f..8a3c70fc0 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 1dae5373` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ dff28e68` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8da63daaa..04615ad30 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 1dae5373` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ dff28e68` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 83e2ba227..74221ed27 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 1dae5373` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ dff28e68` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index e0abdae48..b2da22ad2 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 1dae5373` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ dff28e68` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e4511a81d..02beced86 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 1dae5373` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ dff28e68` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 69dc93820..3ff32519f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 1dae5373` 验证存在 +- 源码校验：以上路径均已在 `main @ dff28e68` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/docs/repository-skills.md b/knowledge/repos/vllm-omni/docs/repository-skills.md index af394fed9..e9e78944b 100644 --- a/knowledge/repos/vllm-omni/docs/repository-skills.md +++ b/knowledge/repos/vllm-omni/docs/repository-skills.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-04 type: rule tags: [vllm-omni, docs] -sources: ["PR #6029", "PR #6046", .claude/skills/readme.md, .claude/skills/add-diffusion-model/SKILL.md, .claude/skills/add-tts-model/SKILL.md, .claude/skills/diffusion-perf-opt/SKILL.md, .claude/skills/precheck-pr/SKILL.md, .claude/skills/precheck-pr/references/checklists.md, .claude/skills/precheck-pr/references/examples-policy.md, .claude/skills/quantization/SKILL.md, .claude/skills/review-pr/SKILL.md, .claude/skills/vllm-omni-npu-upgrade/SKILL.md, .claude/skills/vllm-omni-test/SKILL.md, docs/contributing/README.md] +sources: ["PR #6029", "PR #6046", "PR #6097", .claude/skills/readme.md, .claude/skills/add-diffusion-model/SKILL.md, .claude/skills/add-tts-model/SKILL.md, .claude/skills/diffusion-perf-opt/SKILL.md, .claude/skills/find-simplifications/SKILL.md, .claude/skills/precheck-pr/SKILL.md, .claude/skills/precheck-pr/references/checklists.md, .claude/skills/precheck-pr/references/examples-policy.md, .claude/skills/production-add-diffusion-model/SKILL.md, .claude/skills/production-add-diffusion-model/references/api-and-recipes.md, .claude/skills/production-add-diffusion-model/references/feature-patterns.md, .claude/skills/production-add-diffusion-model/references/performance-patterns.md, .claude/skills/production-add-diffusion-model/references/production-validation.md, .claude/skills/quantization/SKILL.md, .claude/skills/review-pr/SKILL.md, .claude/skills/vllm-omni-npu-upgrade/SKILL.md, .claude/skills/vllm-omni-test/SKILL.md, docs/contributing/README.md] confidence: high --- @@ -19,9 +19,12 @@ confidence: high - 触发：增删/重命名 `.claude/skills/*`、修改 frontmatter `name`，或更新 contributor task-to-skill table 与 `.claude/skills/readme.md`。 - 强制：以目标 revision 的目录与每份 `SKILL.md` frontmatter 为准，分别校验链接路径和 invocation - name。`main @ a15b5b4b` 恰有 8 个目录且都有 `SKILL.md`：`add-diffusion-model`、 - `add-tts-model`、`diffusion-perf-opt`、`precheck-pr`、`quantization`、`review-pr`、 - `vllm-omni-npu-upgrade`、`vllm-omni-test`。NPU 的目录/链接名是 + name。`dff28e6` 恰有 10 个目录且都有 `SKILL.md`：`add-diffusion-model`、 + `add-tts-model`、`diffusion-perf-opt`、`find-simplifications`、`precheck-pr`、 + `production-add-diffusion-model`、`quantization`、`review-pr`、`vllm-omni-npu-upgrade`、 + `vllm-omni-test`。`production-add-diffusion-model` 是 Day-0 vertical slice 完成后的生产化 + workflow；初始 architecture port、registry wiring 与 basic loading 仍先用 + `add-diffusion-model`。NPU 的目录/链接名是 `vllm-omni-npu-upgrade`，frontmatter 与展示的调用名是 `vllm-omni-npu-model-runner-upgrade`，两者不能互换。旧 catalog 的 `add-omni-model` 与 `generate-release-note` 没有目标目录，不能继续列为可用 skill。 @@ -29,9 +32,10 @@ confidence: high registration。contributor guide 的 GitHub 链接指向可变 `main`；发布审计必须读取 pinned target SHA，不能把未来 `main` 内容倒灌到当前合同。 - 验收：对 catalog/task table 的每行做 link-exists + frontmatter-name 对照，并反向检查所有顶层 - skill 目录都被登记。目标提交只改两份 Markdown，没有修改 skill body、reference、script、test - 或 agent runtime；报告的 strict MkDocs/pre-commit 证明文档可构建，不证明 automatic discovery、 - invocation 或行为正确，且没有 catalog link/frontmatter 的自动回归。 + skill 目录都被登记。PR #6097 增加 production skill body、四份 reference、agent descriptor 与 + readme entry；这些文件是 contributor guidance，不证明 automatic discovery、invocation 或生产行为 + 正确。报告的 strict MkDocs/pre-commit 只证明文档可构建，且没有 catalog link/frontmatter 或 + production-combination 的自动回归。 ## DOCSKILL-1b — skill 是可审查指令，不是授权、执行证明或安全边界 @@ -67,3 +71,37 @@ confidence: high - 验收：分别覆盖 add、copy、rename、modify、delete，且 generic-looking model-specific 脚本必须 block；真正 model-neutral、由配置选择模型的 task/protocol entrypoint 可通过。检查两个 skill 的 reference 都解析到 canonical policy，并把结果作为独立 examples-policy 维度报告。^[PR #6046] + +## DOCSKILL-1d — diffusion 生产化必须以逐行证据矩阵而非 Day-0 或 roadmap 声明 + +- 触发：使用 `production-add-diffusion-model`，或声称 diffusion model 的 API/limit parity、FP8、 + DLO、Cache-DiT、attention/operator optimization、batching/step execution、硬件 recipe、性能或 + 生产可靠性已支持。 +- 强制：Day-0 vertical slice 与 production readiness 分开报告。建立至少以 task、API mode、execution + mode/capacity、shape/schedule、attention backend、packed layout、cache policy、quantization、offload + mode、topology、hardware、dtype、output representation/transport 为 key 的矩阵。状态只能是 + `validated`（精确 row 有可复现 correctness 与 deployment artifact）、`limited`（通过范围、限制及 + rejection/fallback 明确）、`unsupported`（稳定 architectural/platform 限制已证明且 fail early），或 + `not tested`（默认；没有充分证据）。每个 `validated` row 记录 model/checkpoint revision、Omni SHA、 + command、request asset/hash、seed/schedule/shape、raw output artifact、environment 与 result；每个 + 非 `validated` row 也记录 reason/evidence。 +- 强制：Function、Accuracy、Performance、Reliability 是独立 readiness tracks，不是新的编号 CI level； + 映射到现有 [L1–L5 taxonomy](../ci/guides/test-tiers.md)：Function 为 L1 logic 加 L2 basic + offline/online E2E（real-model scenarios 扩展至 L3/L4）；Accuracy 为重点 L3、深入/扩展 L4； + Performance 为时间预算允许的 L3 threshold、完整 baseline/regression 的 L4；Reliability 的便宜 + rejection/recovery 可更早跑，而长 soak、fault injection 与 stability 属 weekly L5。topology 是相应 + level 内的 test case，不重新定义 level。 +- 强制：逐 axis 验证；dense BF16 single-device oracle 有 scoped parity 后才能提升，不能从 import、 + server startup、platform abstraction、另一 task/card/topology 或 capability 继承支持。direct-checkpoint + mmap 的当前 TP1/non-HSDP/non-online-quant restriction 不等于 DLO 全局不支持：TP>1 经 ordinary + TP-aware loader 后仍可对 DLO AllGather/no-AllGather 建立独立 scoped row。Day-0 case study、open + optimization roadmap 和未来 capability 均只可作待验证线索，不能写成 model support。 +- 禁止：把 roadmap、实现存在、generic bridge 或 framework capability 写成 production support；把 + `not tested` work queue 提升为 evidence；把另一硬件、量化/loading lane、cache/offload/topology 或 + public streaming 的结果外推到未测组合；把生产 track 或结构性 protocol 当作 latency、throughput、 + cancellation、accuracy 或 stability verdict。 +- 验收：PR/recipe 明确 Day-0 已通过的 representative task 和仍未关闭的 production gate；官方 + advertised task 的 API/offline/online、strict loading、fixed-reference quality 与 negative validation + 分别有证据，recommended deployment row 另有 correctness、isolation、abort/error cleanup、benchmark + 与 soak evidence。所有未验证组合保持 `not tested`，而 Function/Accuracy/Performance/Reliability CI + 分别指向 owner、artifact 和可操作 failure output。^[PR #6097] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 4fd37d5a5..97da887fc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 1dae5373`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ dff28e68`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from ec22c7ac \ - --to 1dae5373 \ + --from 1dae5373 \ + --to dff28e68 \ --repo \ --mode report-only ``` From ed0e6ffa5cd8b35be97586a0ac360807a17c0fc7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:16:16 +0800 Subject: [PATCH 448/465] knowledge(vllm-omni): sync fda3af8b (PR #6878) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/minimax-h3/rules.md | 8 ++++---- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index d41867d97..64af19c7d 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 1dae53731389cce1d2f99697c920a4c159196548 + previous_audited_sha: dff28e686711add0e929ce0c5660b09ef26c8c88 audited_ref: main - audited_sha: dff28e686711add0e929ce0c5660b09ef26c8c88 + audited_sha: fda3af8bbcdcdd822139e139955970157ab44f69 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index d5ab9fa4f..953d43148 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -dff28e686711add0e929ce0c5660b09ef26c8c88`. The canonical machine baseline +fda3af8bbcdcdd822139e139955970157ab44f69`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8a3c70fc0..2e5c37b1e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ dff28e68` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ fda3af8b` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 04615ad30..bb6c436b4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ dff28e68` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ fda3af8b` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 74221ed27..89bb79d4c 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ dff28e68` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ fda3af8b` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b2da22ad2..76c848bbb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ dff28e68` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ fda3af8b` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 02beced86..087669c9f 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ dff28e68` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ fda3af8b` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3ff32519f..23c82f7d5 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ dff28e68` 验证存在 +- 源码校验：以上路径均已在 `main @ fda3af8b` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 97da887fc..6aeda4a69 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ dff28e68`（2026-09-03）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ fda3af8b`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 1dae5373 \ - --to dff28e68 \ + --from dff28e68 \ + --to fda3af8b \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index f35b614c0..ce8a7720a 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", "PR #6040", "PR #6724", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526", "PR #6410", vllm_omni/diffusion/models/minimax_h3/denoise_loop.py] +sources: ["PR #5703", "PR #5706", "PR #5720", "PR #5737", "PR #5764", "PR #5779", "PR #5801", "PR #5824", "PR #5829", "PR #5836", "PR #5837", "PR #5840", "PR #5853", "PR #5881", "PR #5891", "PR #5896", "PR #5991", "PR #5997", "PR #6000", "PR #6040", "PR #6724", benchmarks/diffusion/backends.py, benchmarks/diffusion/diffusion_benchmark_serving.py, docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, vllm_omni/config/model.py, vllm_omni/config/omni_config.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/backends/trtllm_attn.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/cache/teacache/, vllm_omni/diffusion/forward_context.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/lora.py, vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/quality_policy.py, vllm_omni/diffusion/models/minimax_h3/time_request.py, vllm_omni/diffusion/models/minimax_h3/vae.py, vllm_omni/diffusion/offloader/module_residency.py, vllm_omni/diffusion/sched/sigma_schedule.py, vllm_omni/diffusion/utils/hf_utils.py, vllm_omni/entrypoints/omni_base.py, vllm_omni/quantization/int8_config.py, tests/dfx/perf/scripts/run_diffusion_benchmark.py, tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_trtllm_attn.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/cache/test_teacache_extractors.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/diffusion/models/minimax_h3/test_minimax_h3_dlo_lifecycle.py, tests/diffusion/models/minimax_h3/test_minimax_h3_lora.py, tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py, tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py, tests/diffusion/offloader/test_module_residency.py, tests/diffusion/quantization/test_int8_config.py, recipes/MiniMaxAI/MiniMax-H3.md, recipes/MiniMaxAI/MiniMax-H3-5090.md, recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/MiniMaxAI/MiniMax-H3-NPU.md, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, "PR #5915", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6061", "vllm_omni/platforms/npu/models/minimax_h3.py", "vllm_omni/platforms/npu/platform.py", "tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_rope.py", "PR #6167", "PR #6173", "PR #6283", "PR #6281", "PR #6878", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6476", "PR #6526", "PR #6410", vllm_omni/diffusion/models/minimax_h3/denoise_loop.py] confidence: high --- @@ -180,9 +180,9 @@ confidence: high ## MMH3-1m — H3 modulation 融合必须保持操作顺序、精度与平台回退 - 触发：修改 MiniMax H3 的 AdaLN scale/shift、gated residual、`norm1`/`norm2` 融合、最终输出调制、Triton modulation wrapper 或 TeaCache extractor 的对应调用。 -- 强制：保持 H3 的操作顺序：`norm1` 后执行 indexed scale/shift，attention 后执行 gated residual 并紧接 `norm2` 与第二组 indexed affine，最终层继续在投影前调制；CUDA 融合路径使用 FP32 累加并在输出边界转换到目标 dtype。CPU fallback 必须保留旧的两步数值边界，即 RMSNorm 结果先转换为输入 dtype，再执行 affine；所有路径都必须按 `combined_indices` 逐行选择同一 shift、scale 和 gate。 -- 禁止：交换 residual、RMSNorm、affine 或 MLP 的顺序；在 CUDA 融合路径重新引入 BF16 中间舍入后宣称同一精度；让非 CUDA 平台绕过原生 RMSNorm backend；把 CPU fallback 的结果或一次 H3 性能观察描述成 CUDA bit-exact、跨平台 parity 或通用加速保证。 -- 验收：H3 contract test 以 eager 公式分别核对 `norm1` 调制、gated residual+`norm2` 调制和 final-layer 调制，覆盖 index reorder、空行、输出 dtype、CPU 两步 fallback 与 CUDA FP32 容差；NPU 测试必须确认使用 `RMSNorm.forward_npu`/`torch_npu.npu_rms_norm` 而非 Triton，并在固定 shape、steps、seed、硬件和 checkpoint 下单独验证 kernel 数量与性能。^[PR #6281] +- 强制：保持 H3 的操作顺序：`norm1` 后执行 indexed scale/shift，attention 后执行 gated residual 并紧接 `norm2` 与第二组 indexed affine，最终层继续在投影前调制；CUDA Triton 路径先以 FP32 计算 `residual + gate * branch`，必须立即经 BF16 再转回 FP32，并以同一个已舍入值写入 `residual_out` 和执行 RMSNorm；该 round trip 必须留在寄存器，不能增加全局内存中间量。CPU fallback 必须保留旧的两步数值边界，即 RMSNorm 结果先转换为输入 dtype，再执行 affine；所有路径都必须按 `combined_indices` 逐行选择同一 shift、scale 和 gate。 +- 禁止：交换 residual、RMSNorm、affine 或 MLP 的顺序；让 CUDA RMSNorm 消费未写入 residual 的未舍入 FP32 更新值；为该修复增加全局中间 materialization；让非 CUDA 平台绕过原生 RMSNorm backend；把 CUDA BF16 exact test、B300 手工 video A/B 或一次 CI 结果描述成跨平台 parity、广泛质量恢复或通用性能保证。 +- 验收：H3 contract test 以 eager 公式分别核对 `norm1` 调制、gated residual+`norm2` 调制和 final-layer 调制，覆盖 index reorder、空行、输出 dtype 与 CPU 两步 fallback；新增 CUDA+Triton fixed-seed BF16 regression 必须以 `torch.equal` 断言 fused `modulated_out` 等于从返回的 `residual_out` 重算的 `rms_norm_indexed_scale_shift`。NPU 测试必须确认使用 `RMSNorm.forward_npu`/`torch_npu.npu_rms_norm` 而非 Triton；B300 video A/B 仅作手工有界复现，PR head 的 Buildkite vllm-omni build #14512 已失败，不能替代此 CUDA contract test。^[PR #6281] ^[PR #6878] ## MMH3-4a — H3 Qwen3-VL NPU 文本 MLP 必须使用 packed gate/up 与融合 SwiGLU From 5c9bd016615c85e0c65bb00262f41d2d890db205 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:19:35 +0800 Subject: [PATCH 449/465] knowledge(vllm-omni): sync e43c7875 (PR #6359) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 4 +- .../components/diffusion/step-and-batching.md | 25 +++++++++++- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../repos/vllm-omni/models/bagel/_index.md | 8 ++-- .../vllm-omni/models/bagel/architecture.md | 21 +++++++++- .../repos/vllm-omni/models/bagel/rules.md | 38 ++++++++++++++++++- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 13 files changed, 96 insertions(+), 22 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 64af19c7d..9245565b3 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: dff28e686711add0e929ce0c5660b09ef26c8c88 + previous_audited_sha: fda3af8bbcdcdd822139e139955970157ab44f69 audited_ref: main - audited_sha: fda3af8bbcdcdd822139e139955970157ab44f69 + audited_sha: e43c7875445e3b5efc51f2eb5d20080980909914 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 953d43148..4c6fcb5ae 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -fda3af8bbcdcdd822139e139955970157ab44f69`. The canonical machine baseline +e43c7875445e3b5efc51f2eb5d20080980909914`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 2e5c37b1e..0f4031c8c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ fda3af8b` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ e43c7875` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index bb6c436b4..f93fb1333 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ fda3af8b` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ e43c7875` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -44,7 +44,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ | video/audio mux、DLO DP wave、result queue、async pump、SHM ownership、async-output wait 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | | HunyuanImage3 Distil CFG、MeanFlow special-token / scheduler-r 合同 | [HunyuanImage3 HY3-8d](../../models/hunyuan-image3/rules.md) | -| diffusion step 与 request/continuous batching | [step and batching](step-and-batching.md) | +| diffusion step 与 request/continuous batching；DIFF-1af BAGEL image step wave、packed state、geometry/CFG admission 与 fallback cleanup | [step and batching](step-and-batching.md) | | request-wave admission coalescing、stable window、deadline 与 finite config | [admission wait rules](rules-admission-wait.md) | | paged KV/cache 预算、native/backend/platform 闭环、GQA/Ring/Ulysses layout、FlashInfer plan、能力 metadata | [paged cache 与系统运行时规则](rules-system-runtime.md) | | Scheduler-managed diffusion KV 的请求控制面、Hunyuan layout 与未实现边界 | [paged KV control plane](paged-kv-control-plane.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md index 85b233380..79572ac12 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md +++ b/knowledge/repos/vllm-omni/components/diffusion/step-and-batching.md @@ -1,10 +1,10 @@ --- title: "Step 执行合同与 batching 模式" created: 2026-07-16 -updated: 2026-09-02 +updated: 2026-09-05 type: guide tags: [vllm-omni, components, diffusion] -sources: ["PR #5599", "PR #5810", "PR #6045", docs/design/feature/diffusion_continuous_batching.md, docs/user_guide/diffusion/execution_modes.md, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/sched/step_scheduler.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_step_pipeline.py] +sources: ["PR #5599", "PR #5810", "PR #6045", "PR #6359", "PR #7049", docs/design/feature/diffusion_continuous_batching.md, docs/user_guide/diffusion/execution_modes.md, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/sched/request_scheduler.py, vllm_omni/diffusion/sched/step_scheduler.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/test_diffusion_engine.py, tests/diffusion/test_diffusion_step_pipeline.py, tests/diffusion/models/bagel/test_step_execution.py] --- # Step 执行合同与 batching 模式 @@ -62,6 +62,27 @@ step mode 当前拒绝所有 diffusion cache backend，但这不等于拒绝 int runner 对新 admission 先调用 `receive_multi_kv_cache_distributed()`，再 `prepare_encode()`，且有 顺序测试。限制描述必须区分这两个机制。 +## DIFF-1af：BAGEL image step wave 是模型专有的 packed-state 合同 + +**适用范围**：BAGEL 的 image-only step execution，不向其他 pipeline 推导 geometry、CFG 或 +schedule 语义；模型侧完整合同见 [BAGEL-3](../../models/bagel/rules.md#bagel-3step-image-wave-必须保持请求局部状态与兼容边界)。 + +**合同**：BAGEL 的四段 step lifecycle 可将多个兼容 image state 打成一波 DiT forward，但每个 +request 的 latent、timestep/scheduler、KV/CFG context、renormalization 和 output/trajectory 保持 +局部；packed token indexes 要按 request rebase，结果也要按 request split。admission 先为 img2img +求 effective geometry，且仅 geometry 与 BAGEL CFG/renormalization 设置均相同者可同 wave。 +two-stage topology 只让 diffusion Stage 参与，single-stage 显式 text output 保持 complete-request +forward；BAGEL step mode 拒绝 SP 和 diffusion cache backend，但可保留 inter-stage KV transfer。 + +runner 在 finished/aborted step state 后若切到 full-forward wave，必须先清理 state cache、paged +diffusion-KV 与 stale `InputBatch`。BAGEL schedule 以当前模型实现为准：#7049 已将 #6359 的临时 +语义恢复为 `N` schedule points 对应 `N - 1` Euler updates，image path 拒绝 `N < 2`；不要将该模型 +历史 benchmark 或“exactly N updates”泛化成 shared step contract。 + +**验收**：至少两条兼容 image request 覆盖 packed-state isolation、CFG3、index rebasing 与逐请求 +output；再覆盖 geometry/CFG 不兼容不合批、SP/cache rejection、以及 step finish/abort → explicit +text full-forward cleanup。^[PR #6359] ^[PR #7049] + ## 多 document packing 是模型能力，不是 continuous batching 的默认语义 通用 step contract 只要求 scheduler 能交付一组 state；是否能把它们塞进**一次** DiT forward， diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 89bb79d4c..8a4afd1ba 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ fda3af8b` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ e43c7875` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 76c848bbb..daf3dcbe7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ fda3af8b` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ e43c7875` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 087669c9f..e72a8e637 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ fda3af8b` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ e43c7875` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 23c82f7d5..888227382 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ fda3af8b` 验证存在 +- 源码校验：以上路径均已在 `main @ e43c7875` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/bagel/_index.md b/knowledge/repos/vllm-omni/models/bagel/_index.md index 37b51c28e..f7b9b6310 100644 --- a/knowledge/repos/vllm-omni/models/bagel/_index.md +++ b/knowledge/repos/vllm-omni/models/bagel/_index.md @@ -1,10 +1,10 @@ --- title: "BAGEL（统一模型多形态部署参照）" created: 2026-07-21 -updated: 2026-09-02 +updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/model_executor/models/bagel/, vllm_omni/diffusion/models/bagel/, vllm_omni/deploy/bagel.yaml] +sources: ["PR #6359", "PR #7049", vllm_omni/model_executor/models/bagel/, vllm_omni/diffusion/models/bagel/, vllm_omni/deploy/bagel.yaml] --- # BAGEL @@ -44,7 +44,7 @@ sources: [vllm_omni/model_executor/models/bagel/, vllm_omni/diffusion/models/bag | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | KV 桥接、3 路 CFG、MoT、变体拓扑 | [architecture](architecture.md) | AR→DiT 数据流与 reviewer 陷阱 | -| CFG 分支 position ID 打包 | [rules](rules.md) | 1-D RoPE 与 Lance mRoPE 的序列轴合同 | +| CFG 分支 position ID 打包、BAGEL-3 step wave | [rules](rules.md) | 1-D RoPE 与 Lance mRoPE 的序列轴合同；BAGEL 专有 step/packing、有效几何与 fallback 清理 | ## 配置与 checkpoint 差异 @@ -58,5 +58,7 @@ sources: [vllm_omni/model_executor/models/bagel/, vllm_omni/diffusion/models/bag - 一个模型要支持多种 stage 拓扑/思考形态时参考其拓扑与 KV 合同;审查 KV-cache 桥接、CFG 伴随请求或 MoT 权重路径改动时先读 architecture。 +- 调查 BAGEL image step execution 时先读 BAGEL-3：two-stage 只在 diffusion Stage 1 step， + single-stage text 保持完整请求路径；有效 geometry/CFG 兼容性决定能否同 wave。 - [Lance](../lance/_index.md) 的 pipeline 继承自 `BagelPipeline` （`diffusion/models/lance/`,见该页）——改 Bagel 公共面先扫 lance。 diff --git a/knowledge/repos/vllm-omni/models/bagel/architecture.md b/knowledge/repos/vllm-omni/models/bagel/architecture.md index 65026057c..055bc1c1e 100644 --- a/knowledge/repos/vllm-omni/models/bagel/architecture.md +++ b/knowledge/repos/vllm-omni/models/bagel/architecture.md @@ -1,10 +1,10 @@ --- title: "BAGEL 架构" created: 2026-07-21 -updated: 2026-09-02 +updated: 2026-09-05 type: architecture tags: [vllm-omni, models, diffusion] -sources: [vllm_omni/model_executor/models/bagel/bagel.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/model_executor/stage_input_processors/bagel.py] +sources: ["PR #6359", "PR #7049", vllm_omni/model_executor/models/bagel/bagel.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/model_executor/stage_input_processors/bagel.py] --- # BAGEL 架构 @@ -72,6 +72,23 @@ sources: [vllm_omni/model_executor/models/bagel/bagel.py, vllm_omni/diffusion/mo （对齐 Lance 噪声流）;`forward` 单 prompt（多了只取第一个并告警）;分辨率超 `max_latent_size × latent_downsample` 直接 raise。 +## Image step execution 与 wave 边界 + +`prepare_encode` 为单 image request 留下 request-local state，`denoise_step` 可以将兼容 state +pack 为一次 DiT forward，`step_scheduler` 推进各自进度，`post_decode` 逐 request 产出。有效 +img2img geometry 必须在 admission 前从输入图像解析；只有 geometry 与 BAGEL CFG/renormalization +设置一致的请求可共用 wave。packing 后 CFG3 的分支 KV、token index、scheduler state 和可选 +trajectory 仍须逐 request 还原，不能把波内状态当成一个逻辑 request。 + +`bagel`/`bagel_think` 仅 diffusion Stage 1 接入该 lifecycle，AR Thinker 不变； +`bagel_single_stage` 的 image request 可接入，而显式 text2text/img2text 保持完整 `forward()`。 +Step mode 拒绝 SP 与 diffusion cache backend。完成/abort step request 后，runner 须先清掉 state +和 paged diffusion-KV，才可切换到完整 text fallback。详细可执行合同见 +[BAGEL-3](rules.md#bagel-3step-image-wave-必须保持请求局部状态与兼容边界)。 + +不要从 #6359 的历史文档继承 denoise count 或 benchmark 结论：#7049 已恢复 BAGEL 的 `N` schedule +points / `N - 1` Euler update 约定，并拒绝 `N < 2` 的 image request。 + ## 怎样验证功能、精度和性能 - 单元：`tests/diffusion/models/bagel/`（`test_naive_cache`、 diff --git a/knowledge/repos/vllm-omni/models/bagel/rules.md b/knowledge/repos/vllm-omni/models/bagel/rules.md index 340b7f631..586139de7 100644 --- a/knowledge/repos/vllm-omni/models/bagel/rules.md +++ b/knowledge/repos/vllm-omni/models/bagel/rules.md @@ -1,10 +1,10 @@ --- title: "BAGEL 实现规则" created: 2026-09-02 -updated: 2026-09-02 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5775", "PR #5884", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/diffusion/models/lance/lance_transformer.py, tests/diffusion/cache/test_cache_backends.py] +sources: ["PR #5775", "PR #5884", "PR #6359", "PR #7049", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/diffusion/models/lance/lance_transformer.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/models/bagel/test_step_execution.py] confidence: high --- @@ -50,3 +50,37 @@ hook/context 始终绑定同一 nested model，disable 清理安装时记录的 offload 分别验证同一 object identity 和 missing-middle 行为。当前 mock 测试覆盖 dotted enable/refresh/summary；multi-target disable 虽有 mock 覆盖，但没有真实 cross-consumer、异常或 worker shutdown 证据。^[PR #5884] + +## BAGEL-3：Step image wave 必须保持请求局部状态与兼容边界 + +**适用范围**：`BagelPipeline` 的 `prepare_encode` → `denoise_step` → +`step_scheduler` → `post_decode` image step lifecycle，以及 runner 在 step wave 和完整 +`forward()` fallback 间切换；不把 AR Thinker 或 text generation 纳入该合同。 + +**合同**：每个 image request 在 `prepare_encode` 建立自己的 latent、timestep/scheduler progress、 +主/CFG KV context、CFG 参数与可选 trajectory；同一 wave 可以把兼容 request 的 DiT 输入、KV 和 +CFG 分支打包进一次 denoise forward，但必须重 base 每个 request 的 packed token index，之后按 +request 拆回 noise prediction、scheduler update 和输出。三路 CFG（gen/cfg_text/cfg_img）仍是 +每 request 的语义，不能因 packing 混用 KV、renormalization 或 trajectory。 + +Admission 必须先解析有效图像几何：img2img 从输入图像推导、对齐 latent stride 并受 checkpoint +size limit 限制，不能直接按用户的 `height`/`width` 合批。一个 step wave 只可含 effective geometry +相同且 `cfg_text_scale`、`cfg_img_scale`、`cfg_interval`、`cfg_renorm_type`、 +`cfg_renorm_min` 相同的请求；其余请求独立排队。该支持只覆盖 image generation：两个 two-stage +拓扑的 Stage 1 diffusion 使用 step protocol、Stage 0 Thinker 仍为 AR；single-stage 的显式 +text2text/img2text 继续走完整请求路径。BAGEL step mode 不支持 sequence parallelism 或 diffusion +cache backend；这不否定 two-stage 的 inter-stage KV transfer。 + +调度步数须以当前 BAGEL schedule builder 为准：#6359 的一度行为已由 #7049 恢复为 `N` 个 +schedule points、`N - 1` 个 Euler denoise updates，所有 image path 对 `N < 2` 失败。不要把旧的 +“exactly N updates”、one-step 支持或该 PR 的 benchmark 当成当前合同。 + +step request abort/finish 后，runner 必须先清理 request state、paged diffusion-KV 和 stale +`InputBatch`，再 dispatch 后续完整 `forward()` 的 text fallback；否则先前 wave 的 tensor/KV 可 +保留并污染生命周期。 + +**验收**：以两个重叠 image request 覆盖 packed index rebasing、各自 CFG/renorm、state isolation +和 output ownership；再覆盖 img2img effective geometry 不同而不合批、SP/cache rejection，以及 +step request finish/abort 后的 explicit text full-forward fallback。schedule regression 必须断言 +`N < 2` 在 complete 和 step image path 均被拒绝，且不将 denoise update 数写成 `N`。^[PR #6359] +^[PR #7049] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 6aeda4a69..7a9c1fab8 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ fda3af8b`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ e43c7875`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from dff28e68 \ - --to fda3af8b \ + --from fda3af8b \ + --to e43c7875 \ --repo \ --mode report-only ``` From 424185a70d2d735cf531ad5bdcc1ef9ed4a641bd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:22:57 +0800 Subject: [PATCH 450/465] knowledge(vllm-omni): sync 4179b0ca (PR #6354) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 4 +- .../serving/rules-session-lifecycle.md | 16 +++++++- .../vllm-omni/components/serving/rules.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 14 +++---- .../models/nemotron-voicechat/_index.md | 10 +++-- .../models/nemotron-voicechat/rules.md | 37 ++++++++++++++++++- 13 files changed, 75 insertions(+), 24 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9245565b3..0273dd267 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: fda3af8bbcdcdd822139e139955970157ab44f69 + previous_audited_sha: e43c7875445e3b5efc51f2eb5d20080980909914 audited_ref: main - audited_sha: e43c7875445e3b5efc51f2eb5d20080980909914 + audited_sha: 4179b0cae13dba0ef535da829d83feb209ae6a0d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 4c6fcb5ae..cc22929b8 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -e43c7875445e3b5efc51f2eb5d20080980909914`. The canonical machine baseline +4179b0cae13dba0ef535da829d83feb209ae6a0d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 0f4031c8c..ee97c8ffd 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ e43c7875` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 4179b0ca` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f93fb1333..0ae5d4c06 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ e43c7875` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 4179b0ca` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 8a4afd1ba..1002c427b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ e43c7875` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 4179b0ca` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index daf3dcbe7..23bc52443 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ e43c7875` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 4179b0ca` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index e72a8e637..539a3c4a7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ e43c7875` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 4179b0ca` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 888227382..1842bb7e0 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ e43c7875` 验证存在 +- 源码校验：以上路径均已在 `main @ 4179b0ca` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 @@ -35,7 +35,7 @@ sources: [] | batched chat frontend fan-out、choice cardinality、whole-batch error/cancellation | [batch chat rules](rules-batch-chat.md) | | engine startup/shutdown、stage lifecycle、acknowledged abort 的 shutdown race、duplex stage prewarm/CFG companion，或 opt-in event-driven orchestration / reader-poller reconcile / final-output drain | [engine 生命周期规则](rules-engine-lifecycle.md) | | duplex stage-keyed segment state、跨 stage boundary/metadata 隔离 | [duplex 编排规则](rules-duplex-orchestration.md) | -| adapter-owned full-duplex session、deferred cognition/gate、barge-in cancellation、response ownership、`to_thread` worker drain 或 close/flush/session cleanup | [session lifecycle 规则](rules-session-lifecycle.md) | +| adapter-owned full-duplex session、deferred cognition/gate、barge-in cancellation、response ownership、`to_thread` worker drain、close/flush/session cleanup，或 startup warmup/realtime admission | [session lifecycle 规则](rules-session-lifecycle.md) | | acknowledged abort 的 RPC ACK、queue/router shutdown race、cancellation cleanup 或 backpressure/timeout 保持 | [acknowledged abort 生命周期规则](rules-abort-lifecycle.md) | | assembled FastAPI route 覆盖、method/path ownership 或 request-body forwarding | [app assembly 规则](rules-app-assembly.md) | | stage config→EngineArgs projection、explicit `devices`、TP/local-DP/PP、replica layout 或 worker 创建前的 layout guard | [stage 启动与设备布局规则](rules-stage-startup.md) | diff --git a/knowledge/repos/vllm-omni/components/serving/rules-session-lifecycle.md b/knowledge/repos/vllm-omni/components/serving/rules-session-lifecycle.md index afb4483f1..0347470b8 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-session-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-session-lifecycle.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-05 type: rule tags: [vllm-omni, components, serving] -sources: ["PR #6537", vllm_omni/experimental/fullduplex/core/adapter.py, vllm_omni/experimental/fullduplex/core/runtime.py, vllm_omni/experimental/fullduplex/mage_vl/adapter.py, vllm_omni/experimental/fullduplex/mage_vl/serving/backend.py, vllm_omni/experimental/fullduplex/mage_vl/serving/server.py, tests/e2e/features/fullduplex/test_mage_vl_adapter.py, tests/e2e/features/fullduplex/test_mage_vl_serving.py] +sources: ["PR #6537", "PR #6354", vllm_omni/config/stage_config.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/experimental/fullduplex/core/adapter.py, vllm_omni/experimental/fullduplex/core/runtime.py, vllm_omni/experimental/fullduplex/mage_vl/adapter.py, vllm_omni/experimental/fullduplex/mage_vl/serving/backend.py, vllm_omni/experimental/fullduplex/mage_vl/serving/server.py, tests/e2e/features/fullduplex/test_mage_vl_adapter.py, tests/e2e/features/fullduplex/test_mage_vl_serving.py, tests/e2e/online_serving/nemotron_voicechat_realtime_duplex.py] confidence: high --- @@ -30,3 +30,17 @@ confidence: high explicit close 可 flush pending gate 再完成 response，disconnect 则 cancel；没有 lifecycle method 的既有 adapter 行为不变。PR 的 CPU suite 证明 lifecycle contract；报告的 RTX 4090 run 只是 single-run functional evidence，不得外推 throughput、latency 或一般 hardware support。^[PR #6537] + +## SERV-6i — optional duplex startup warmup 必须闭合 readiness、真实 transport 与 shutdown lifecycle + +- 触发：duplex session startup warmup、`warmup_frames` 配置、realtime WebSocket admission，或 API-server + shutdown 中的 warmup task 管理。 +- 强制：`warmup_frames=0` 保持禁用；启用时先确认 `/health`，再用标记过的 self WebSocket 连接穿过实际 + `/v1/realtime` handler 和生产 session/stage path，完成 configured silent frames 后显式发送 session close。 + 真实 client 在 warmup event 前等待，最多 120 seconds；无论成功、health timeout、缺 optional client + dependency、protocol failure 或异常，warmup coroutine 都必须 set event。失败只记录并继续提供服务； + server shutdown 必须 cancel 未完成的 warmup task。 +- 禁止：用直接调用 model/adapter 代替实际 tagged WS path；bare WebSocket disconnect 后留下占用 session + slot；因 warmup failure 拒绝服务或永久阻塞 real client；或在 shutdown 遗留 background warmup task。 +- 验收：覆盖 disabled、health-then-self-WS ordering、warmup tag 绕过 admission gate、explicit close、 + success/failure 均 set event、real-client 120-second escape hatch，以及 shutdown cancellation。^[PR #6354] diff --git a/knowledge/repos/vllm-omni/components/serving/rules.md b/knowledge/repos/vllm-omni/components/serving/rules.md index 2e1756096..b0256cfa2 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules.md +++ b/knowledge/repos/vllm-omni/components/serving/rules.md @@ -48,7 +48,7 @@ confidence: high | `chat-multimodal-contract` | chat template kwargs、SDK flatten、text/audio response shape | `SERV-4c`（见 [请求输入合同](rules-request-input.md)）+ 命中模型规则 | | `endpoint-capability` | endpoint restriction、route/app-state guard、公开 400 | `SERV-4c`, `SERV-4d` 见 [请求输入合同](rules-request-input.md)；`SERV-5d` 见 engine lifecycle；`SERV-5s` 见 [app assembly](rules-app-assembly.md) | | `engine-lifecycle` | pause/resume、sleep/wake、partial stage/tag、ACK、generation admission、abort cleanup/shutdown race、streaming raw terminal、event-driven orchestration、factory 状态矩阵、TTS adapter detection、replica membership/fault isolation | `SERV-5a`–`SERV-5e`、`SERV-5g`–`SERV-5r`（见 [engine 生命周期规则](rules-engine-lifecycle.md)）、[SERV-5t](rules-abort-lifecycle.md#serv-5t-acknowledged-abort-在-shutdown-边界只忽略已识别的-transport-closure)，`SERV-5f` | -| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion、stage-keyed segment lifecycle，或 adapter-owned deferred cognition/session cleanup | `SERV-6a`–`SERV-6f` 见 [engine 生命周期规则](rules-engine-lifecycle.md)；`SERV-6g` 见 [duplex 编排规则](rules-duplex-orchestration.md)；`SERV-6h` 见 [session lifecycle](rules-session-lifecycle.md) | +| `full-duplex` | duplex opt-in、stage prewarm/fence、async-chunk、CFG companion、stage-keyed segment lifecycle，adapter-owned deferred cognition/session cleanup，或 startup warmup/realtime admission | `SERV-6a`–`SERV-6f` 见 [engine 生命周期规则](rules-engine-lifecycle.md)；`SERV-6g` 见 [duplex 编排规则](rules-duplex-orchestration.md)；`SERV-6h`、`SERV-6i` 见 [session lifecycle](rules-session-lifecycle.md) | | `request-contract` | 请求字段、来源、冲突、dispatcher、consumer view、stage sampling constraints、serialization bounds、transform artifact ownership | `SERV-4a`–`4h`, `SERV-4l`–`4q`，全部见 [请求输入合同](rules-request-input.md) | | `batch-chat-contract` | frontend fan-out、identity、choice cardinality、error/cancellation | `SERV-4i`–`4k`，见 [batch chat rules](rules-batch-chat.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `SERV-0a`, `SERV-0b` | diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 7a9c1fab8..37952b252 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -4,13 +4,13 @@ created: 2026-07-16 updated: 2026-09-05 type: guide tags: [vllm-omni, models] -sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/, "PR #5885", "PR #6727"] +sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/, "PR #5885", "PR #6354", "PR #6727"] --- # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ e43c7875`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 4179b0ca`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -45,9 +45,9 @@ and regular/Turbo deployment profiles. It preserves bare H3's fused single-stage the split topology is selected only by its explicit deploy config. ^[PR #5885] PR #6089 新增 `nemotron_labs_voicechat_duplex.yaml`，因此只增加 deploy profile；它没有 -增加 AR/diffusion registry architecture 或 `OMNI_PIPELINES` key。该 profile 是原生全双工 -Nemotron VoiceChat 的单会话 eager fp32 配置，不能从 YAML 或 PR 的作者测试外推 wall-clock -realtime、确定性 barge-in 或一般硬件支持。^[PR #6089] +增加 AR/diffusion registry architecture 或 `OMNI_PIPELINES` key。PR #6354 将既有 offline、 +streaming 与 duplex 三份 Nemotron VoiceChat deploy profile 改为各自的 fast defaults，但未新增 deploy +YAML、模型族、AR/diffusion registry architecture 或 `OMNI_PIPELINES` key。^[PR #6089] ^[PR #6354] PR #6634 adds Omni-DuplexEval benchmark support without changing the AR registry, diffusion registry, `OMNI_PIPELINES`, or deploy-YAML inventory. Its MiniCPM-o duplex endpoint is an @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from fda3af8b \ - --to e43c7875 \ + --from e43c7875 \ + --to 4179b0ca \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md b/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md index 54c795fae..495543548 100644 --- a/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md +++ b/knowledge/repos/vllm-omni/models/nemotron-voicechat/_index.md @@ -1,10 +1,10 @@ --- title: "NVIDIA Nemotron-Labs VoiceChat" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: index tags: [vllm-omni, models, serving] -sources: ["PR #6089", vllm_omni/deploy/nemotron_labs_voicechat.yaml, vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/asr/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py] +sources: ["PR #6089", "PR #6354", vllm_omni/deploy/nemotron_labs_voicechat.yaml, vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/, vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/asr/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py] confidence: high --- @@ -18,7 +18,7 @@ confidence: high ## 源码路径 -共 30 个文件： +共 32 个文件： - `vllm_omni/deploy/nemotron_labs_voicechat.yaml` - `vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml` @@ -27,6 +27,8 @@ confidence: high - `vllm_omni/model_executor/models/nemotron_voicechat/` - `vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/` - `vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/asr/` +- `vllm_omni/model_executor/models/nemotron_voicechat/talker_graph.py` +- `vllm_omni/model_executor/models/nemotron_voicechat/talker_native.py` - `vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py` ## 依赖的共享代码模块 @@ -55,4 +57,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| 原生 duplex、帧/工具合同与验证边界 | [Nemotron VoiceChat rules](rules.md) | +| 原生 duplex、帧/工具合同与验证边界；native talker geometry/replay/batch output；有界 perception window | [Nemotron VoiceChat rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md b/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md index 28a04e72e..155a4bde2 100644 --- a/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md +++ b/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, serving] -sources: ["PR #6089", vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/experimental/fullduplex/openai/, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py, tests/e2e/features/fullduplex/nemotron_voicechat/, tests/e2e/online_serving/test_nemotron_voicechat_duplex.py, "PR #6831"] +sources: ["PR #6089", "PR #6354", vllm_omni/deploy/nemotron_labs_voicechat.yaml, vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml, vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml, vllm_omni/experimental/fullduplex/nemotron_voicechat/, vllm_omni/experimental/fullduplex/openai/, vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/model_executor/models/nemotron_voicechat/, vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py, tests/e2e/features/fullduplex/nemotron_voicechat/, tests/e2e/online_serving/nemotron_voicechat_realtime_duplex.py, tests/model_executor/models/test_nemotron_voicechat_perception_window.py, tests/model_executor/models/test_nemotron_voicechat_talker_replay.py, tests/model_executor/stage_input_processors/test_nemotron_voicechat.py, "PR #6831"] confidence: high --- @@ -40,3 +40,38 @@ confidence: high - 禁止：用非空 bytes、约 1 LSB 的 RMS 或文本成功证明音频可用；把作者在 H200 实测、generic nightly selector 或 YAML 存在写成 Buildkite pass、wall-clock realtime、确定性 barge-in 或跨硬件保证。不得把 240 秒写成生产模型设置、把 `FINISHED_ERROR` 称作 client error delivery，或声称 #6816 已修复。 - 验收：PR 作者报告 targeted H200 suite 306 passed，reviewer 后续报告 399 CPU tests、offline WAV byte-identical 和 native duplex non-silent audio；这些是提交时证据，当前 KB validation 不能复现真实 checkpoint/H200 E2E。Buildkite 改动只把既有 AMD-ready 步骤的超时从 20 分钟增至 45 分钟，不新增该模型专属 pipeline。^[PR #6089] - stall diagnostic 的验收固定 server 240/client 300，复现 stall 时 server 先记录 full request ID 并标记 `FINISHED_ERROR`，之后 bare client `TimeoutError` 才到期；同时正常 run 仍有 audio。reviewer 已复现该 ordering，仍未证明 client error event。^[PR #6831] + +## NVC-1e — native talker 的 prompt、replay 与 batch 输出必须保持逐 request 的 frame feedback + +- 触发：修改 native talker、Thinker→Talker input processor、stage processor dispatch、talker prompt/ + sampling budget，或 KV recomputation / multi-session output splitting。 +- 强制：native talker 的 placeholder prompt 必须覆盖实际 speaker prefill；shipped Aria geometry 为 + `talker_init_len=37`，故 `max_model_len=16384` 时 budget 为 `max_tokens=16347`。processor 只在 + 具名 `next_stage_hf_config` 参数存在时传入该 stage config，同时按 `streaming_context` 或 + `_streaming_context` 参数名保留既有 bridge state。duplex native producer 每次最多使一个新 timeline + step 可调度，live decode 按帧严格顺序。KV recomputation 必须从保留的 code history 重建 embeds： + `t=1` 使用 `initial_code`，后续 `t` 使用 `codes_rows[t-2]`；replay 不采样、不设置 live pending + step，也不推进没有 rewind 的 unconditional CFG stream。batch output 必须按 request 对齐并为无新 + codes 的 slot 保留其自身 placeholder/cumulative view，供 generic splitter 按 index 分发。 +- 禁止：以第四个位置参数猜测 processor 能力而把 streaming context 传给 `next_stage_hf_config`；让 + prompt 加 generation budget 超过 model length；让 coalesced duplex timeline 越过尚未执行的 frame； + replay 时重采样、推进 unconditional stream，或从别的 request 复用 code tensor。 +- 验收：覆盖 37-token prefill 与 16347 budget、具名 config/context dispatch、one-step duplex drip 和 + batch-aligned outputs。replay tests 必须覆盖 pure replay、replay 后接一个 live step、跨 prefill + boundary、live-first boundary rejection、short-history 与 outpaced-position guards，并断言 + `initial_code` / `codes_rows[t-2]` arithmetic 和 unconditional stream 在 replay 中不变。^[PR #6354] + +## NVC-1f — duplex perception 的滚动 mel window 必须保持 full-history 切片语义 + +- 触发：修改 cache-aware perception、80 ms duplex frame ingestion、streaming mel slice、request + re-open/reset，或长 session 的 audio/mel retention。 +- 强制：rolling window 记录其 stream-global mel column origin，只保留当前 chunk 的 pre-encode cache + 所需范围及足以隔离 reflect-pad/preemphasis 边界效应的前置 margin；slice 以 global geometry 计算后 + 再映射回本地 window。对相同输入，这个有界窗口的 chunk/cache 结果必须等同 full-history mel 的 + 对应切片。若已排队 append 或 stage request re-open 导致 model session 中途缺失，则从当前 input + frame 重新建立 perception state 并记录 warning。 +- 禁止：每个 frame 对全部历史音频重新 featurize；把 trimmed window 的 local column 当作 stream + origin；或因 mid-stream reset / fused prefill 判错令整个 engine core 失败。 +- 验收：以长序列覆盖 window trim 后的 global-origin slice 与 full-history reference 的逐块等价， + 包括 cache、margin 和首帧边界；覆盖 mid-stream session reset/re-open 后从当前 frame 恢复，以及 + prefill branch 由 engine position 而非 duplex sequence 决定。^[PR #6354] From 1dfe32f0e67272d2d4332996380143d25a831a87 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:29:16 +0800 Subject: [PATCH 451/465] knowledge(vllm-omni): sync 8d98cc0c (PR #6817) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/benchmark/_index.md | 1 + knowledge/repos/vllm-omni/benchmark/rules.md | 48 +++++++++++++++++++ .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +-- 11 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 knowledge/repos/vllm-omni/benchmark/rules.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0273dd267..6549a1d63 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: e43c7875445e3b5efc51f2eb5d20080980909914 + previous_audited_sha: 4179b0cae13dba0ef535da829d83feb209ae6a0d audited_ref: main - audited_sha: 4179b0cae13dba0ef535da829d83feb209ae6a0d + audited_sha: 8d98cc0cb7be1b7f75056572093174a97f42962d # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index cc22929b8..e0a93f4f6 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -4179b0cae13dba0ef535da829d83feb209ae6a0d`. The canonical machine baseline +8d98cc0cb7be1b7f75056572093174a97f42962d`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/benchmark/_index.md b/knowledge/repos/vllm-omni/benchmark/_index.md index 3c17b247e..21ace0b0f 100644 --- a/knowledge/repos/vllm-omni/benchmark/_index.md +++ b/knowledge/repos/vllm-omni/benchmark/_index.md @@ -21,6 +21,7 @@ sources: [benchmarks/tts/bench_tts.py, benchmarks/tts/model_configs.yaml, "PR #6 | 遇到什么 | 查看哪里 | |---|---| +| 新增/审查 perf JSON、warmup 覆盖、realtime artifact 或 Hub dataset fallback | [benchmark rules](rules.md) | | 查看旧 benchmark 总览 | [overview](overview.md) | | 运行 AR graph、Hunyuan 或 MiniCPM-o Omni-DuplexEval benchmark | [benchmark guides](guides/_index.md) | | 调查 profiling、模型加载和性能验证错误 | [benchmark incidents](incidents/_index.md) | diff --git a/knowledge/repos/vllm-omni/benchmark/rules.md b/knowledge/repos/vllm-omni/benchmark/rules.md new file mode 100644 index 000000000..68ef8fa39 --- /dev/null +++ b/knowledge/repos/vllm-omni/benchmark/rules.md @@ -0,0 +1,48 @@ +--- +title: "vLLM-Omni Benchmark 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, benchmark] +sources: ["PR #6817", tests/dfx/perf/scripts/run_benchmark.py, tests/benchmarks/test_omniinteract.py] +confidence: high +--- + +# vLLM-Omni Benchmark 规则 + +只有 `BENCH-数字字母` 是可审计规则 ID。证据口径和远端 scope lock 见 +[evidence gate](guides/evidence-gate.md)；模型专有的 MiniCPM-o 行为见 +[MiniCPM-o 4.5 rules](../models/minicpm-o-4-5/rules.md)。 + +## BENCH-1a — perf JSON 的显式 warmup 覆盖必须保留零值 + +- 触发：新增或修改 `tests/dfx/perf/tests` 的 perf JSON，或修改它传给 benchmark client 的 + warmup 解析。 +- 强制：仅在字段缺失时使用 runner 默认值；显式的非负整数（包括 `0`）原样传递到 + client。将测量目标标为冷请求时，在 JSON 明示 `num_warmups: 0`，而不是依赖并发数或 + truthiness 推导。 +- 禁止：用 `value or default`、`max(default, value)` 或类似逻辑把显式零值改成默认 + warmup 数。 +- 验收：metadata/unit test 同时断言字段缺失得到默认值、`num_warmups: 0` 仍为零，且负数、 + bool 或非整数被拒绝。 ^[PR #6817] + +## BENCH-1b — realtime workload 的完成、artifact 与质量资格分层 + +- 触发：runner 为 realtime/OmniInteract 一类 workload 增加 PASS/FAIL 断言或汇总字段。 +- 强制：先断言计划请求全部完成，再断言 workload 汇总的 total、success、failed 与计划 + 一致以及必需 artifacts 完整；把 official manifest eligibility 作为单独报告的质量信号。 + LISTEN-only 合法路径可以没有 response audio 或 transcript chunk。 +- 禁止：用 manifest eligibility、非空 WAV 帧数、或是否有回应音频替代 transport/lifecycle + 成功；也不能在 artifact 缺失或 request 失败时继续产出性能结果。 +- 验收：完整 lifecycle/artifact fixture 通过；缺 summary、请求计数不符、失败请求或 + `artifacts_complete=false` 的 fixture 必须失败；clipped/cancelled 等不合资格输出仍能被 + 报告为本地 lifecycle 成功。 ^[PR #6817] + +## BENCH-1c — checked-in 外部 benchmark workload 必须可复现 + +- 触发：把 Hub dataset 或媒体 archive 写入 checked-in 本地 perf JSON。 +- 强制：使用 `org/repo@immutable-revision` 形式的 dataset 标识，固定 subset、请求顺序和 + 输出目录；filesystem fallback 同时支持 plain `org/repo` 与带 revision 的路径。 +- 禁止：让已提交的性能 workload 跟随 Hub 默认分支，或只测试不带 revision 的 fallback。 +- 验收：配置中的 dataset 标识含 revision；fallback unit coverage 覆盖 plain 与 revision + 两种输入；每个固定 subset 的 case 数、并发与 warmup 设置可由 JSON 直接审计。 ^[PR #6817] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index ee97c8ffd..e50efe7aa 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 4179b0ca` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 8d98cc0c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 0ae5d4c06..46b465b2e 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 4179b0ca` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 8d98cc0c` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1002c427b..70f307b4d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 4179b0ca` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 8d98cc0c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 23bc52443..31150b3fb 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 4179b0ca` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 8d98cc0c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 539a3c4a7..ee53445a2 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 4179b0ca` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 8d98cc0c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1842bb7e0..8c41cab29 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 4179b0ca` 验证存在 +- 源码校验：以上路径均已在 `main @ 8d98cc0c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 37952b252..1c68da958 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 4179b0ca`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 8d98cc0c`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from e43c7875 \ - --to 4179b0ca \ + --from 4179b0ca \ + --to 8d98cc0c \ --repo \ --mode report-only ``` From 823db93927755d5170e9251111edeef72b9c3808 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:31:33 +0800 Subject: [PATCH 452/465] knowledge(vllm-omni): sync 175dec11 (PR #7000) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../repos/vllm-omni/models/ltx2/rules.md | 24 +++++++++++++++++-- 10 files changed, 34 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 6549a1d63..45e7ba1a2 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 4179b0cae13dba0ef535da829d83feb209ae6a0d + previous_audited_sha: 8d98cc0cb7be1b7f75056572093174a97f42962d audited_ref: main - audited_sha: 8d98cc0cb7be1b7f75056572093174a97f42962d + audited_sha: 175dec11bea2a1fb6f69ec2d4034aa3d924cc4ca # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index e0a93f4f6..526b58529 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -8d98cc0cb7be1b7f75056572093174a97f42962d`. The canonical machine baseline +175dec11bea2a1fb6f69ec2d4034aa3d924cc4ca`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index e50efe7aa..07b478504 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 8d98cc0c` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 175dec11` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 46b465b2e..3ff35b23c 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 8d98cc0c` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 175dec11` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 70f307b4d..a124a4bde 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 8d98cc0c` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 175dec11` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 31150b3fb..b2fc81af7 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 8d98cc0c` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 175dec11` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ee53445a2..69ef6ae6d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 8d98cc0c` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 175dec11` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8c41cab29..f626fa5e9 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 8d98cc0c` 验证存在 +- 源码校验：以上路径均已在 `main @ 175dec11` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 1c68da958..18155e02a 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 8d98cc0c`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 175dec11`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 4179b0ca \ - --to 8d98cc0c \ + --from 8d98cc0c \ + --to 175dec11 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/ltx2/rules.md b/knowledge/repos/vllm-omni/models/ltx2/rules.md index 023e8ed87..6797ae8c8 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/rules.md +++ b/knowledge/repos/vllm-omni/models/ltx2/rules.md @@ -1,10 +1,10 @@ --- title: "LTX-2.5 DiffVAE 规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, models, ltx2, diffusion] -sources: ["PR #6189", recipes/LTX/LTX-2.5.md, vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_latents.py, vllm_omni/diffusion/models/ltx2/ltx2_request.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_vae.py] +sources: ["PR #6189", "PR #7000", recipes/LTX/LTX-2.5.md, vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_latents.py, vllm_omni/diffusion/models/ltx2/ltx2_request.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_output_cuda.py, tests/diffusion/models/ltx2/test_ltx2_pipeline.py, tests/diffusion/models/ltx2/test_ltx2_vae.py] confidence: high --- @@ -40,3 +40,23 @@ confidence: high placement 改变固定 seed 的 tile noise，或用 ConvVAE 的 distributed-state 判定 DiffVAE collective。 - 验收：覆盖 early geometry rejection 与 ConvVAE control、tile positive/non-tile negative modes、serial RNG-order parity，以及 output/non-output rank 在 collective/non-collective decode 的分支。^[PR #6189] + +### LTX-3 — LTX decoded-video transport 必须在设备端量化且保持 presentation 与 profiler 边界 + +- 触发：修改 LTX-2/2.3/2.5 decoded-video finalization、`output_type="np"`、worker/IPC transport，或 + LTX diffusion-pipeline profiler wrapping。 +- 强制：fast path 只接受 floating-point RGB `BCTHW` decoded video；detach 后仅当 + `video_processor.config.do_normalize` 为真时在 decoder output dtype 中 denormalize，始终 defensive + clamp 到 `[0, 1]`，再在该 dtype 中 scale/round 并直接转为 contiguous `uint8` `BTHWC`，随后由既有 + post-process boundary 一次 `detach().cpu().numpy()` 交付。该 boundary 只接受 contiguous RGB `BTHWC` + uint8；非 `np` output type 继续走 `VideoProcessor.postprocess_video`。profiler targets 必须同时覆盖 + `video_processor.postprocess_video` 与 `_prepare_video_output_for_transport`；I2V 替换 video processor 后， + profiler 启用时必须重新 wrap 前者。 +- 禁止：为量化 materialize 全帧 `.float()` CUDA temporary、改变 latent/generation，或让 non-`np` output + 走 uint8 fast path；不得把 `do_normalize=False` 的 out-of-range clamp 当作 legacy parity，或把 H200 + 的精确 shape/配置性能结果、LTX similarity matrix 外推为一般硬件、吞吐或兼容性结论。 +- 验收：CPU 覆盖 dtype/normalization、shape/dtype/contiguity、无效 BCTHW/RGB/floating contract 与 profiler + rewrap；CUDA 覆盖 BF16/FP16、normalize true/false 的 device-side contiguous BTHWC uint8 和一次 D2H + 对比。有效 `[0, 1]` 的 `do_normalize=False` 输入须与 unclipped legacy path 比较；另测 intentional + out-of-range defensive clamp。FP32 presentation bytes 必须 exact，BF16/FP16 相对 legacy FP32 + presentation 最多相差一个 uint8 level；官方 similarity 仅是本 PR 所报 LTX matrix 的限定证据。^[PR #7000] From cfa495f0db7e44b8b5bcdfed712705aca4fc0218 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:33:54 +0800 Subject: [PATCH 453/465] knowledge(vllm-omni): sync 0e903e96 (PR #6968) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- .../vllm-omni/models/boogu-image/_index.md | 4 ++-- .../vllm-omni/models/boogu-image/rules.md | 21 ++++++++++++++++++- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 34 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 45e7ba1a2..e7ee44221 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 8d98cc0cb7be1b7f75056572093174a97f42962d + previous_audited_sha: 175dec11bea2a1fb6f69ec2d4034aa3d924cc4ca audited_ref: main - audited_sha: 175dec11bea2a1fb6f69ec2d4034aa3d924cc4ca + audited_sha: 0e903e9695a68817b9817885e834cd7c02148aae # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 526b58529..c2650d6c0 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -175dec11bea2a1fb6f69ec2d4034aa3d924cc4ca`. The canonical machine baseline +0e903e9695a68817b9817885e834cd7c02148aae`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 07b478504..16b19e438 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 175dec11` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0e903e96` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3ff35b23c..a8d333801 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 175dec11` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 0e903e96` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a124a4bde..a3d212b8b 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 175dec11` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0e903e96` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b2fc81af7..83a68a2ef 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 175dec11` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0e903e96` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 69ef6ae6d..9c82643e0 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 175dec11` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0e903e96` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index f626fa5e9..e8d91900c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 175dec11` 验证存在 +- 源码校验：以上路径均已在 `main @ 0e903e96` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/boogu-image/_index.md b/knowledge/repos/vllm-omni/models/boogu-image/_index.md index 991d63b9a..57a455965 100644 --- a/knowledge/repos/vllm-omni/models/boogu-image/_index.md +++ b/knowledge/repos/vllm-omni/models/boogu-image/_index.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: index tags: [vllm-omni, models, diffusion] -sources: ["PR #6701", "PR #6571", "PR #6786", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/diffusion/models/boogu_image/, vllm_omni/entrypoints/utils.py] +sources: ["PR #6701", "PR #6571", "PR #6786", "PR #6968", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/diffusion/models/boogu_image/, vllm_omni/entrypoints/utils.py] confidence: high --- @@ -51,4 +51,4 @@ confidence: high | 遇到什么 | 查看哪里 | |---|---| -| revision/component loader、Edit-Turbo checkpoint、real RoPE/platform selection，或 Edit CFG branch/rank/VAE decode 合同 | [Boogu rules](rules.md#boogu-1c-edit-cfg-分支rank-分派与-vae-decode-必须保持语义闭合) | +| revision/component loader、Edit-Turbo checkpoint、real RoPE/platform selection、Edit CFG branch/rank/VAE decode 合同，或 request-level T2I batching、TI2I batch=1 gate、request-major mask/generator/output mapping | [Boogu rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/boogu-image/rules.md b/knowledge/repos/vllm-omni/models/boogu-image/rules.md index 724c444fc..b1e2382ea 100644 --- a/knowledge/repos/vllm-omni/models/boogu-image/rules.md +++ b/knowledge/repos/vllm-omni/models/boogu-image/rules.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #6701", "PR #6571", "PR #6786", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py, vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py, vllm_omni/entrypoints/utils.py, tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py, tests/diffusion/models/boogu_image/test_boogu_image_transformer.py, tests/entrypoints/test_utils.py, tests/e2e/online_serving/test_boogu_image_edit.py] +sources: ["PR #6701", "PR #6571", "PR #6786", "PR #6968", docs/models/supported_models.md, recipes/Boogu/Boogu-Image.md, vllm_omni/config/omni_config.py, vllm_omni/diffusion/models/boogu_image/pipeline_boogu_image.py, vllm_omni/diffusion/models/boogu_image/boogu_image_transformer.py, vllm_omni/entrypoints/utils.py, tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py, tests/diffusion/models/boogu_image/test_boogu_image_transformer.py, tests/entrypoints/test_utils.py, tests/e2e/online_serving/test_boogu_image_edit.py] confidence: high --- @@ -41,3 +41,22 @@ confidence: high - 验收：有界测试覆盖 Base/single Edit 两分支、double-stream Edit 三分支、CFG2 的 `rank0=[0,2]`/`rank1=[1]`、CFG3 一 rank 一 branch、精确 guidance 合并式、CFG-off 每 rank positive-only，以及 CUDA VAE efficient→math fallback；不将这些测试外推为 TP/SP/HSDP、cache、offload 或通用性能结论。^[PR #6786] + +## BOOGU-1d — request batching 只能合并 T2I，并保持请求行与输出的对应关系 + +- 触发：修改 `BooguImagePipeline` 的 request batching、pre-process compatibility key、prompt/mask + reshape、generator collate 或 request-level output split。 +- 强制：纯文本或无 reference image 的 T2I 必须写入稳定的 + `("boogu_image", "t2i")` compatibility key 后才可合批；带 reference image 的 TI2I 必须把 + `request_id` 放进 key，并在 `has_reference and req.num_reqs > 1` 时 fail-close。TI2I 保持 + batch=1，直到 scheduler key 能区分 `guidance_scale_2_provided`：该字段目前不在 + `RequestBatchSamplingParamsKey` 中，而 `forward` 会从 batch 首请求读取它。request-local + generators 按 request-major/output-minor 合并；`num_outputs_per_prompt` 的结果必须拆回每个 + 原始请求一个 `DiffusionOutput`。prompt mask 用 `repeat_interleave` 保持与 reshaped embeds 相同的 + request-major 行序。 +- 禁止：让 T2I/TI2I 共用 key、以首请求的 guidance mode 代表 edit batch、把 TI2I 作为已安全 + 合批，或把不同 physical batch shape 的数值差异说成跨请求值泄漏或 serial-equivalence。 +- 验收：用真实 `DiffusionRequestBatch` 和 scheduler key builder 覆盖 stable T2I/unique TI2I key、 + batched TI2I raise、B=2/N=2 distinct-mask 行序、request-major generators 和每请求 output slice； + CFG B=2 isolation 至少分别改变 partner 的 positive prompt、negative prompt 和 seed。CPU/mock + tests 不证明 GPU CFG、TI2I batch safety、吞吐或 QPS/perf claim。^[PR #6968] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 18155e02a..5b10cca74 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 175dec11`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0e903e96`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 8d98cc0c \ - --to 175dec11 \ + --from 175dec11 \ + --to 0e903e96 \ --repo \ --mode report-only ``` From 5f8607bbde8d468867dde32718e091b7abd73cc1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:39:08 +0800 Subject: [PATCH 454/465] knowledge(vllm-omni): sync 10701e18 (PR #6989) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 4 ++-- .../components/diffusion/rules-output-lifecycle.md | 12 ++++++++++++ .../repos/vllm-omni/components/diffusion/rules.md | 6 +++--- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 28 insertions(+), 16 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index e7ee44221..c3525ce7e 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 175dec11bea2a1fb6f69ec2d4034aa3d924cc4ca + previous_audited_sha: 0e903e9695a68817b9817885e834cd7c02148aae audited_ref: main - audited_sha: 0e903e9695a68817b9817885e834cd7c02148aae + audited_sha: 10701e18389fe282ca8c2510cd2022a76a9dad2a # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index c2650d6c0..3fa763eab 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0e903e9695a68817b9817885e834cd7c02148aae`. The canonical machine baseline +10701e18389fe282ca8c2510cd2022a76a9dad2a`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 16b19e438..47e98cc40 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0e903e96` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 10701e18` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index a8d333801..574e98ddd 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0e903e96` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 10701e18` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -41,7 +41,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ | PEFT 与 distilled LoRA、startup fusion、delta/key/alpha、Qwen/Wan transformer mapping | [LoRA rules](rules-lora.md) | | Blackwell FA4、显式/auto CUDA backend、per-role mask/SP pad；local FlashAttention deterministic opt-in、role-aware masked cross-attention、NPU packed mask-free/laser fallback、TRTLLM packed-padding 与 config propagation | [attention rules](rules-attention.md) | | SP auto-padding、`mask_sp_padding`、dense/varlen 与 advanced UAA 边界 | [SP padding rules](rules-sp-padding.md) | -| video/audio mux、DLO DP wave、result queue、async pump、SHM ownership、async-output wait 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | +| video/audio mux、DLO DP wave、RPC reply ownership/result release、result queue、async pump、SHM ownership、async-output wait 与 shutdown | [output/runtime rules](rules-output-lifecycle.md) | | distilled continuous sigma schedule、boundary/step 语义与 modality shift | [sigma schedule rules](sigma-schedules.md) | | HunyuanImage3 Distil CFG、MeanFlow special-token / scheduler-r 合同 | [HunyuanImage3 HY3-8d](../../models/hunyuan-image3/rules.md) | | diffusion step 与 request/continuous batching；DIFF-1af BAGEL image step wave、packed state、geometry/CFG admission 与 fallback cleanup | [step and batching](step-and-batching.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md index 7520b1b39..9fbc53f9d 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-output-lifecycle.md @@ -144,6 +144,18 @@ confidence: high - 禁止：只在 topology 标 video 但 formatter 仍发 image；要求每个 stage 都是 diffusion 才允许 video final；遗漏已注册 video alias 后静默回 image；用 ndarray 形状猜语义，或把局部 closure 作为跨进程 hook。 - 验收：逐个 canonical class/alias 断言 default 与 multi-stage final type，video/image endpoint 正负 capability 及最终 output type；action 另验证 post-process callable 可跨目标进程边界使用，最终 output 保留 stable key、shape、dtype 和有限性。PR #6953 的 CPU metadata unit test 只断言两种 SANA class 均分类为 video；PR #6847 的同类 CPU metadata test 只覆盖显式 LTX distilled two-stage alias 分类。两者都不是新的 endpoint、真实模型或 online/offline E2E 证据。^[PR #4222] ^[PR #5885] ^[PR #6847] ^[PR #6953] +## DIFF-4z — 全 rank RPC 必须执行，但仅 reply owner 保留输出 + +- 触发：RPC 在所有 rank 执行，而 `output_rank` 或 DP replica primary 决定唯一回复者；结果可能 + 携带 device-resident diffusion 输出。 +- 强制：所有应执行 rank 都调用目标方法以维持 collective 语义；`should_reply=false` 时在 + `_worker_busy_loop` 绑定返回值之前返回 `(None, False)`，只有回复 rank 保留并 enqueue 原结果。 +- 禁止：因不回复而跳过执行，或把非回复 rank 的 device-resident 结果交回 busy loop 并保留到下次 + 请求覆盖。 +- 验收：非回复 rank 断言目标方法确实执行且返回 `None, False`；回复 rank 断言返回同一对象。测试 + 必须覆盖实际采用的 reply-selection 形式。当前证据只覆盖 CPU 单元级返回所有权，不证明 GPU + 显存下降或真实多进程回收时序。^[PR #6989] + ## DIFF-9a — 共享 PyAV 预构造帧 mux 必须闭合资源与音频时间零点 - 触发：修改共享 diffusion 的预构造 PyAV 视频帧 mux、可选音频 mux、container cleanup 或编码器异常传播。 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index 95bf94536..c0391ac06 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -4,7 +4,7 @@ created: 2026-07-20 updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", "PR #5831", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150", "PR #6410", tests/diffusion/layers/test_activation.py, tests/diffusion/layers/test_fused_qk_norm_rope_npu.py] +sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", "PR #5831", "PR #6989", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, tests/diffusion/test_multiproc_engine_concurrency.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150", "PR #6410", tests/diffusion/layers/test_activation.py, tests/diffusion/layers/test_fused_qk_norm_rope_npu.py] confidence: high --- @@ -33,7 +33,7 @@ confidence: high | LPIPS/PSNR/相似度阈值、CPU offload、量化质量证据 | `quality-evidence`：`DIFF-3a` | changed exact case → runner `execute_model` → model pipeline；A/B 同路径 | | Wan VAE spatial shard、gather/trim/reshard、empty tail、attention extent | `distributed-vae`：`DIFF-3b` | `distributed/autoencoders/wan_spatial_shard.py` → patched decoder attention/conv → gather final frame | | paged KV/cache、backend/platform、GQA/layout、Ring/Ulysses、SymmMem A2A、FlashInfer quant、masked cross-attention、预算与 admission | `system-runtime`：`DIFF-4a`–`4j`, `DIFF-4v`, `DIFF-4x` | engine init → metadata/config → attention parallel/backend → platform hook → scheduler/serving | -| worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 raise/reply/status 路径 | +| worker/RPC 异常、rank-status、traceback/device cache 清理 | `system-runtime`：`DIFF-4d`；reply ownership 见 [output/runtime](rules-output-lifecycle.md) `DIFF-4z` | `diffusion_worker.py::{_execute_rpc,_worker_busy_loop}` 的 execute/reply/status 路径 | | worker title、log prefix、DP/PP/SP/CFG/TP/FS/RP/EP local rank | [worker observability](rules-worker-observability.md)：`DIFF-4w` | `diffusion_worker.py::_setup_diffusion_worker_proc_title_and_log_prefix` → `distributed/parallel_state.py` group getters | | image/diffusion service time、forward time、missing-versus-zero、per-step metric | [`metrics-evidence`](rules-metrics.md)：`DIFF-5a`, `DIFF-5b` | output formatter/sampling metadata → stage stats aggregation → serving Prometheus emission | | image/video/audio/action final type、registry alias、endpoint capability、formatter | [`output-runtime`](rules-output-lifecycle.md#diff-1s-diffusion-output-type-必须从模型声明闭合到-topology-与-formatter)：`DIFF-1s` | `model_metadata.py` → default/multi-stage config → video capability → `output_formatter.py` | @@ -46,7 +46,7 @@ confidence: high | `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s`, `2y`, `2z` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r`, `2w`, `12a` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o`, `2x` 见 [LoRA](rules-lora.md) | | `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | | `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、SymmMem A2A、能力 metadata、异常与并发 | `DIFF-4a`–`4j`、`4v`、`4x`（`4a`–`4i`、`4x` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)，`4v` 见 [attention rules](rules-attention.md)） | +| `system-runtime` | cache/预算、native/backend/platform、attention layout、SymmMem A2A、能力 metadata、异常与并发 | `DIFF-4a`–`4j`、`4v`、`4x`（`4a`–`4i`、`4x` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)，`4v` 见 [attention rules](rules-attention.md)）；reply ownership `4z` 见 [output/runtime](rules-output-lifecycle.md) | | [`metrics-evidence`](rules-metrics.md) | diffusion timing、step normalization、missing-versus-zero、benchmark interpretation | `DIFF-5a`, `DIFF-5b` | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index a3d212b8b..ef7363613 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0e903e96` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 10701e18` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 83a68a2ef..1316a4c47 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0e903e96` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 10701e18` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 9c82643e0..859eb737e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0e903e96` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 10701e18` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index e8d91900c..2389696cc 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0e903e96` 验证存在 +- 源码校验：以上路径均已在 `main @ 10701e18` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 5b10cca74..d8e6641e2 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0e903e96`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 10701e18`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 175dec11 \ - --to 0e903e96 \ + --from 0e903e96 \ + --to 10701e18 \ --repo \ --mode report-only ``` From 38f4e5b80464943afbd527bf8022a9f7c762dcef Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:41:18 +0800 Subject: [PATCH 455/465] knowledge(vllm-omni): sync 41f4f9d8 (PR #6983) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 4 ++-- .../components/diffusion/rules-platform-runtime.md | 12 ++++++++++-- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index c3525ce7e..bb3544008 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0e903e9695a68817b9817885e834cd7c02148aae + previous_audited_sha: 10701e18389fe282ca8c2510cd2022a76a9dad2a audited_ref: main - audited_sha: 10701e18389fe282ca8c2510cd2022a76a9dad2a + audited_sha: 41f4f9d8c91a15b6d2f3b65516bfd885559d8e27 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 3fa763eab..7d2da7fb4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -10701e18389fe282ca8c2510cd2022a76a9dad2a`. The canonical machine baseline +41f4f9d8c91a15b6d2f3b65516bfd885559d8e27`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 47e98cc40..56880b49d 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 10701e18` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 41f4f9d8` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 574e98ddd..1a493ade9 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 10701e18` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 41f4f9d8` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 @@ -33,7 +33,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ | image/diffusion timing、step normalization、missing-versus-zero 与 benchmark 解释边界 | [metrics evidence rules](rules-metrics.md) | | vLLM/torch rebase、MoE/quant helper 漂移、kernel backend capability 与 matched accuracy | [upstream 兼容规则](rules-upstream-compat.md) | | worker process title、拓扑局部 rank 与日志前缀 | [worker observability rules](rules-worker-observability.md) | -| 平台 IR-op priority、Inductor/eager 默认顺序与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | +| 平台 IR-op priority、Inductor/eager 默认顺序、pipeline compile 准入与模型 hook 合并 | [platform runtime rules](rules-platform-runtime.md) | | CustomOp XPU platform dispatch、PyTorch-native fallback、CUDA contract 与非同义 override 边界 | [CustomOp dispatch rules](rules-custom-op-dispatch.md) | | runtime temporary 与 loader-scoped parameter dtype、shared RMSNorm accuracy | [tensor dtype rules](rules-tensor-dtype.md) | | Wan VAE height/width spatial reshard、empty tail、attention extent | [Wan spatial-shard rules](rules-wan-spatial-shard.md) | diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md b/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md index 1a4705649..13fff5632 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-platform-runtime.md @@ -1,10 +1,10 @@ --- title: "Diffusion 平台运行时规则" created: 2026-09-02 -updated: 2026-09-04 +updated: 2026-09-05 type: rule tags: [vllm-omni, components, diffusion] -sources: ["PR #6058", vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/platforms/musa/platform.py, "PR #6267", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py"] +sources: ["PR #6058", vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/platforms/musa/platform.py, "PR #6267", "vllm_omni/diffusion/models/ltx2/ltx2_phase_adapter.py", "PR #6983", vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py] confidence: high --- @@ -18,6 +18,7 @@ confidence: high | PR 描述在做什么 | 规则 | 第一批 live 源码 | |---|---|---| | IR-op priority、Inductor/eager 默认顺序、模型 hook 覆盖 | `DIFFPLAT-1a` | platform `get_default_ir_op_priority` → `diffusion_worker._resolve_ir_op_priority` → model `get_diffusion_ir_op_priority_func` | +| pipeline `setup_compile()` 的平台准入 | `DIFFPLAT-3a` | `DiffusionModelRunner` → `pipeline.setup_compile()` | ## DIFFPLAT-1a — platform IR-op priority 必须区分 Inductor 与 eager @@ -40,3 +41,10 @@ confidence: high - 强制：当 `current_platform.is_rocm()` 且 `input_.device.type == "cpu"` 时，必须使用 `F.linear(input_, weight, bias)`，绕过没有 CPU kernel 的 `rocm_unquantized_gemm`；其他设备路径继续遵循既有 dispatch 和 batch-invariant 逻辑。 - 禁止：仅因 host platform 是 ROCm 就把 CPU tensor 交给 ROCm GEMM；不得把该 fallback 推广为 ROCm kernel 的通用 CPU 支持或改变其他平台的 dispatch 语义。 - 验收：ROCm host 的 CPU 输入测试必须断言结果来自 `F.linear` 且未调用 `rocm_unquantized_gemm`；同时回归 ROCm device、CUDA/其他平台及既有 batch-invariant 路径。^[PR #6267] + +## DIFFPLAT-3a — runner 必须是 pipeline compile 的唯一平台准入者 + +- 触发：修改 `DiffusionModelRunner` 的 `setup_compile()` 调用条件，或模型 pipeline 的 `setup_compile()`。 +- 强制：runner 仅在 `enforce_eager` 为 false 且 `current_omni_platform.supports_torch_inductor()` 时调用 pipeline `setup_compile()`；pipeline 只负责其模型专有的编译工作，沿用 runner 的平台能力决策。 +- 禁止：pipeline 以 `torch.cuda.is_available()` 或其他 CUDA 专属条件重复收窄该入口；不得仅据此变更宣称 XPU、ROCm 或其他平台已经完成运行时验证。 +- 验收：以 mock platform 覆盖 eager、Inductor 支持与不支持的组合，断言仅合格组合调用 `setup_compile()`；并回归 DreamZero 的 setup 路径经 runner 可达。^[PR #6983] diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ef7363613..ceb53b409 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 10701e18` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 41f4f9d8` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 1316a4c47..9de569045 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 10701e18` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 41f4f9d8` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 859eb737e..d941171c7 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 10701e18` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 41f4f9d8` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 2389696cc..3b2a0f5ee 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 10701e18` 验证存在 +- 源码校验：以上路径均已在 `main @ 41f4f9d8` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d8e6641e2..79ce518fc 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 10701e18`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 41f4f9d8`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 0e903e96 \ - --to 10701e18 \ + --from 10701e18 \ + --to 41f4f9d8 \ --repo \ --mode report-only ``` From 3cfefe05d9e2ec93f241ed4ddef8bf508be6380f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:44:40 +0800 Subject: [PATCH 456/465] knowledge(vllm-omni): sync 7493cbd3 (PR #7019) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/qwen-omni/_index.md | 8 ++++++-- knowledge/repos/vllm-omni/models/qwen-omni/rules.md | 9 ++++++++- 11 files changed, 26 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index bb3544008..8624e9424 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 10701e18389fe282ca8c2510cd2022a76a9dad2a + previous_audited_sha: 41f4f9d8c91a15b6d2f3b65516bfd885559d8e27 audited_ref: main - audited_sha: 41f4f9d8c91a15b6d2f3b65516bfd885559d8e27 + audited_sha: 7493cbd3c24befd9628fe679ebc65f4c00a57701 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7d2da7fb4..9cc004efd 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -41f4f9d8c91a15b6d2f3b65516bfd885559d8e27`. The canonical machine baseline +7493cbd3c24befd9628fe679ebc65f4c00a57701`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 56880b49d..b32608e1f 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 41f4f9d8` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 7493cbd3` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 1a493ade9..f5645205a 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 41f4f9d8` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 7493cbd3` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index ceb53b409..1f9f4d58f 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 41f4f9d8` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 7493cbd3` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 9de569045..337206e09 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 41f4f9d8` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 7493cbd3` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d941171c7..31d30c063 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 41f4f9d8` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 7493cbd3` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 3b2a0f5ee..39f588220 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 41f4f9d8` 验证存在 +- 源码校验：以上路径均已在 `main @ 7493cbd3` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 79ce518fc..407f217c1 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 41f4f9d8`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 7493cbd3`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 10701e18 \ - --to 41f4f9d8 \ + --from 41f4f9d8 \ + --to 7493cbd3 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md index d5bc16976..0e48f9b7e 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/_index.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/_index.md @@ -4,7 +4,7 @@ created: 2026-07-16 updated: 2026-09-05 type: index tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322", "PR #6886", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/quantization/component_config.py, docs/design/qwen3_omni_tts_performance_optimization.md] +sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322", "PR #6886", "PR #7019", vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/registry.py, vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/engine/stage_init_utils.py, vllm_omni/quantization/component_config.py, tests/engine/test_stage_engine_args.py, docs/design/qwen3_omni_tts_performance_optimization.md] --- # Qwen-Omni 家族（Qwen2.5-Omni / Qwen3-Omni / Qwen3-TTS） @@ -30,6 +30,10 @@ sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322" Thinker/Talker/Code2Wav 不得再次改名，仅在自身声明 packed modules 时补充对应 metadata。 component/default quant configs 都是同一映射合同的一部分；验收规则归本目录的 Qwen-Omni rules。^[PR #5687] +- Qwen3-Omni 的 MoE backend 默认仅在已解析值仍为 `auto`/缺失时选择 `triton`，显式 backend + 保留；随仓库发布的 2×H100 profile 为 Thinker 与 Talker 都 pin `moe_backend: triton`，不再以 + `VLLM_USE_FLASHINFER_MOE_FP16` 这类环境变量选择。legacy/typed builder 的最终 args 都是验收 + 边界；精确规则与证据限制见本目录 Qwen-Omni rules。^[PR #7019] - vLLM 0.27 后 upstream 与 Omni registry 有同名 architecture：Omni 必须覆盖全局 entry；plain `vllm serve` 缺 `model_stage` 时 Qwen2.5/Qwen3 默认 thinker，Qwen3 thinker 在非 staged 模式返回 bare tensor，只有 staged talker consumer 才请求 captured layers；共享验收见 @@ -56,4 +60,4 @@ sources: ["PR #5073", "PR #5671", "PR #5687", "PR #5976", "PR #6284", "PR #4322" | 遇到什么 | 查看哪里 | |---|---| | stage 拓扑、代际差异与官方性能优化结论 | [architecture](architecture.md) | -| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary、AWQ/compressed-tensors 名称映射、固定种子音频回归、audio-encoder head/TP divisibility，或 Qwen2.5 code2wav soft-fail/旧 speech helper | [Qwen-Omni rules](rules.md) | +| Qwen3-Omni Thinker MRoPE、CUDA compilation custom-op boundary、AWQ/compressed-tensors 名称映射、MoE backend default/override、固定种子音频回归、audio-encoder head/TP divisibility，或 Qwen2.5 code2wav soft-fail/旧 speech helper | [Qwen-Omni rules](rules.md) | diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index 2488d2b38..ec204096d 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, qwen-omni] -sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", "PR #6886", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py] +sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", "PR #6886", "PR #7019", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/engine/stage_init_utils.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/engine/test_stage_engine_args.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py] confidence: high --- @@ -65,3 +65,10 @@ confidence: high - 强制：`_codec_to_audio` 在 `token2wav` 缺失时必须直接返回 `None`；`_init_token2wav_model` 只能由持有已解析 `hf_model_folder` 的 Token2Wav weight-loading 路径调用；语音生成维持 Thinker → Talker → Code2Wav 的 stage handoff。 - 禁止：无参数调用 `_init_token2wav_model()`；恢复 `generate_speech` 或 `_convert_to_codec_tokens`；仅 thread `sampling_metadata` 就声称修复旧 helper（其 Talker `compute_logits` 调用仍带已失效的额外参数）；或让单一 wrapper 违反 staged ownership 而同时拥有 Talker 和 Token2Wav。 - 验收：目标树中不存在两个已删除 helper 或无参数 initializer call；`token2wav is None` 的 `_codec_to_audio` 路径返回 `None` 而非 `TypeError`；Token2Wav load path 仍向 initializer 传入解析后的 HF directory。PR 未新增测试或 E2E，故这些是静态/call-site 合同，不构成运行时、音频质量或性能证据。^[PR #6886] + +## QOMNI-1f — Qwen3-Omni MoE 默认值必须走显式 backend 合同 + +- 触发：修改 Qwen3-Omni Thinker/Talker 的 MoE backend、`qwen3_omni_moe.yaml`、stage-engine argument finalization，或迁移旧的环境变量 workaround。 +- 强制：仅当 `model_arch` 为 `Qwen3OmniMoeForConditionalGeneration` 且已解析的 `moe_backend` 为缺失或 `auto` 时，finalization 才默认写入 `triton`；任何显式 backend 必须原样保留。随仓库发布的 `qwen3_omni_moe.yaml` 必须在 Thinker stage 0 和 Talker stage 1 都显式 pin `moe_backend: triton`；Code2Wav 不继承该模型专属 MoE 设置。 +- 禁止：通过 `VLLM_USE_FLASHINFER_MOE_FP16` 或其他 process-wide 环境变量选择 backend；把默认值写成覆盖用户/deploy 显式选择；只改 deploy YAML 却不覆盖 legacy 与 typed builder 的最终 engine args。 +- 验收：以 Qwen3-Omni default 和一个显式 backend control，分别经 `build_legacy_engine_args_dict` 与 `build_engine_args_dict_from_omni_stage_config` 断言最终 `moe_backend`；默认两路径均为 `triton`，显式值不变。helper 单测只能补充分支覆盖，不能代替 builder-level assertion。PR 只提供 compile/Ruff/diff 检查及因缺少 `torch` 未运行的 targeted pytest，故不构成 GPU 稳定性、性能或音频质量证据。^[PR #7019] From ca378f63751fde3992695a705d30356350e5e4ba Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:46:33 +0800 Subject: [PATCH 457/465] knowledge(vllm-omni): sync 22cefe04 (PR #7020) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- knowledge/repos/vllm-omni/models/ltx2/rules.md | 14 +++++++++----- 10 files changed, 21 insertions(+), 17 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8624e9424..9eb818874 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 41f4f9d8c91a15b6d2f3b65516bfd885559d8e27 + previous_audited_sha: 7493cbd3c24befd9628fe679ebc65f4c00a57701 audited_ref: main - audited_sha: 7493cbd3c24befd9628fe679ebc65f4c00a57701 + audited_sha: 22cefe04b4007607ff235ea8520cd404ad44df3e # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9cc004efd..776383002 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -7493cbd3c24befd9628fe679ebc65f4c00a57701`. The canonical machine baseline +22cefe04b4007607ff235ea8520cd404ad44df3e`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index b32608e1f..5db492615 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 7493cbd3` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 22cefe04` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index f5645205a..8c9300944 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 7493cbd3` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 22cefe04` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1f9f4d58f..161358a53 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 7493cbd3` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 22cefe04` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 337206e09..6ac23b7d3 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 7493cbd3` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 22cefe04` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 31d30c063..a388b9a4e 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 7493cbd3` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 22cefe04` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 39f588220..ea5db7152 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 7493cbd3` 验证存在 +- 源码校验：以上路径均已在 `main @ 22cefe04` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 407f217c1..3f07cdceb 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 7493cbd3`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 22cefe04`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 41f4f9d8 \ - --to 7493cbd3 \ + --from 7493cbd3 \ + --to 22cefe04 \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/ltx2/rules.md b/knowledge/repos/vllm-omni/models/ltx2/rules.md index 6797ae8c8..d683c2378 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/rules.md +++ b/knowledge/repos/vllm-omni/models/ltx2/rules.md @@ -4,7 +4,7 @@ created: 2026-09-04 updated: 2026-09-05 type: rule tags: [vllm-omni, models, ltx2, diffusion] -sources: ["PR #6189", "PR #7000", recipes/LTX/LTX-2.5.md, vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_latents.py, vllm_omni/diffusion/models/ltx2/ltx2_request.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx2_output_cuda.py, tests/diffusion/models/ltx2/test_ltx2_pipeline.py, tests/diffusion/models/ltx2/test_ltx2_vae.py] +sources: ["PR #6189", "PR #7000", "PR #7020", recipes/LTX/LTX-2.5.md, requirements/common.txt, docs/user_guide/diffusion/attention_backends/huggingface_hub.md, vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_latents.py, vllm_omni/diffusion/models/ltx2/ltx2_request.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx25_pipeline.py, tests/diffusion/models/ltx2/test_ltx2_output_cuda.py, tests/diffusion/models/ltx2/test_ltx2_pipeline.py, tests/diffusion/models/ltx2/test_ltx2_vae.py] confidence: high --- @@ -13,19 +13,23 @@ confidence: high ### LTX25-1 — LTX-2.5 的 decoder、统计量与 artifact 必须作为同一版本化合同维护 - 触发：修改 LTX-2.5 full/distilled one- or two-stage pipeline 的视频 decoder、`ltx2_use_conv_vae`、 - latent normalization、Native sidecar 或 component residency。 + latent normalization、Native sidecar、component residency 或 NATTEN Hub-kernel dependency/remediation + guidance。 - 强制：LTX-2.5 默认构造 canonical Native DiffVAE；仅布尔 `ltx2_use_conv_vae=true` 可在启动时 opt into ConvVAE，I2V encoding 仍使用 ConvVAE。provided latent 的 normalize 与 decode 前的 denormalize 必须从实际视频 decoder 取得 mean、std 和 scaling factor。Native decoder 与独立 sidecar 先查 model root；Hub fallback 使用 artifact repository 自己的 pinned revision，不能把 - Diffusers model revision 跨仓库复用。DiffVAE 初始化必须要求 runtime `kernels==0.14.1`、支持的 - NATTEN Hub kernel 和可操作的失败说明。 + Diffusers model revision 跨仓库复用。DiffVAE 初始化必须要求 runtime `kernels==0.15.2`、支持的 + NATTEN Hub kernel 和可操作的失败说明；该 pin 必须与支持范围 + `transformers >=5.10.1, <5.15` 同步维护。 - 禁止：让 ConvVAE statistics 处理 DiffVAE latent；将非布尔 opt-in 静默转换；把 local primary model path 误当成独立 sidecar 必须离线；使用未 pin 的 canonical Native artifact，或向用户建议不存在的 production FlexAttention fallback。 - 验收：覆盖 full/distilled profile 的默认和 ConvVAE opt-in、非布尔拒绝、Native decoder strict conversion、active-decoder statistics、local-first/repository-scoped revision routing，以及缺少 NATTEN - runtime prerequisite 的 actionable error。^[PR #6189] + runtime prerequisite 的 actionable error；`requirements/common.txt`、Hugging Face Hub installation + guidance、production error 和 LTX-2.5 test 必须一致地说明 `kernels==0.15.2` 与上述 Transformers + 范围。^[PR #6189] ^[PR #7020] ### LTX25-2 — DiffVAE 请求边界与分布式 decode 必须显式 fail-closed From ee23fad7eafe2c3c6175c049ef44c20016d7585b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:50:35 +0800 Subject: [PATCH 458/465] knowledge(vllm-omni): sync 0f9ee6af (PR #6947) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 2 +- knowledge/repos/vllm-omni/ci/rules.md | 14 +++++++++++++- .../vllm-omni/components/configuration/_index.md | 2 +- .../repos/vllm-omni/components/diffusion/_index.md | 2 +- .../vllm-omni/components/distributed/_index.md | 2 +- .../vllm-omni/components/model-executor/_index.md | 2 +- .../repos/vllm-omni/components/scheduler/_index.md | 2 +- .../repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 11 files changed, 26 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 9eb818874..33ab868fb 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 7493cbd3c24befd9628fe679ebc65f4c00a57701 + previous_audited_sha: 22cefe04b4007607ff235ea8520cd404ad44df3e audited_ref: main - audited_sha: 22cefe04b4007607ff235ea8520cd404ad44df3e + audited_sha: 0f9ee6afd83c5e1dc02fe834d8d2c4c51ecd4d3b # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 776383002..041210b76 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -22cefe04b4007607ff235ea8520cd404ad44df3e`. The canonical machine baseline +0f9ee6afd83c5e1dc02fe834d8d2c4c51ecd4d3b`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index 05ec03f17..d9133ff30 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -208,7 +208,7 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie | 遇到什么 | 查看哪里 | |---|---| -| 审查硬件 lane（含 MiniMax-H3 DLO DP2 ready smoke 的证据边界）、回归 fence、CI 工具供应链或 xdist/shared fixture | [CI rules](rules.md) | +| 审查硬件 lane（含 MiniMax-H3 DLO DP2 ready smoke 的证据边界）、回归 fence、CI 工具供应链、ASR 文本比较或 xdist/shared fixture | [CI rules](rules.md) | | AMD/ROCm timeout、quarantine、memory signal、Qwen3-TTS argv 或 Qwen3-Omni control-plane fixture | [AMD/ROCm CI rules](rules-amd.md) | | CUDA L4 Kubernetes preset、GPU-count shard、resource/retry policy | [L4 Kubernetes CI rules](rules-l4-k8s.md) | | 查看仓库特有 CI 陷阱 | [CI guides](guides/_index.md) | diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 2654529d4..01a45b7eb 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -4,7 +4,7 @@ created: 2026-08-23 updated: 2026-09-05 type: rule tags: [vllm-omni, ci] -sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818", "PR #6830", "PR #6884", tests/diffusion/conftest.py, tests/diffusion/attention/test_flash_attn.py, tests/buildkite/test_amd_pipeline.py, tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py] +sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818", "PR #6830", "PR #6884", tests/diffusion/conftest.py, tests/diffusion/attention/test_flash_attn.py, tests/buildkite/test_amd_pipeline.py, tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py, "PR #6947"] confidence: high --- @@ -25,6 +25,7 @@ confidence: high | 重模型 cold start、共享 engine/server fixture、sleep/wake | `OMNI-CI-2c` | `tests/entrypoints/test_omni_sleep_mode.py`、OmniServer fixture scope/lock | | diffusion tiny builder、model settings、alignment exclusion、重模型 OOM | `OMNI-CI-2d` | `tests/model_tests/diffusion/{diff_model_builders,model_settings,test_alignment}.py` → common offline tests | | perf baseline、hardware label、DFX result artifact、assert-baseline | `OMNI-CI-3a` | `tests/dfx/conftest.py`、`tests/dfx/perf/scripts/run_benchmark.py`、`run_diffusion_benchmark.py`、`tests/dfx/perf/tests/**` | +| ASR transcript、文本相似度或词汇归一化 | `OMNI-CI-3d` | `tests/helpers/media.py::{preprocess_text, cosine_similarity_text}` → `tests/helpers/tests/test_media.py` | | AMD/ROCm CI timeout、quarantine、diagnostic hang、memory assertion、Qwen3-TTS argv 或 Qwen3-Omni sleep/abort control plane | [OMNI-CI-2f](rules-amd.md#omni-ci-2f-amd-ci-stabilization-必须保留有界执行与测量信号) | `.buildkite/amd/**` → `tests/buildkite/test_amd_pipeline.py` / `ci/qwen3_omni_moe_colocate_async.yaml` → target test | ## OMNI-CI-1a — 硬件 lane 必须真实收集并执行目标路径 @@ -310,3 +311,14 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload 通过外推 XPU。 - 验收：核对 image provenance、XPU import/startup，以及这三条路径 compile 未被调用而 其他模型保留原能力。^[PR #5957] + +## OMNI-CI-3d — ASR 文本比较先归一化已证实的等价词形 + +- 触发：修改 ASR transcript 与期望文本的相似度评分，或修改其词汇归一化表。 +- 强制：在 n-gram 余弦评分前，仅对有复现证据的等价词形做大小写无关、词边界保护的 + 归一化；回归测试同时断言预处理结果相等和最终相似度合同。 +- 禁止：降低相似度阈值来吞掉格式差异；把已证实的转写缩写差异宣称为模型音质回归； + 未经证据把其他模型的转换表或推测词形并入共享评分器。 +- 验收：`"eight kilohertz"` 与 `"8 kHz"` 归一化后一致，且 + `cosine_similarity_text(...)` 约等于 `1.0`；焦点 helper 回归测试在 CPU 上通过。 + ^[PR #6947] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5db492615..5ee1ea01c 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 22cefe04` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 0f9ee6af` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 8c9300944..03523a97b 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 22cefe04` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 0f9ee6af` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 161358a53..580419d60 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 22cefe04` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 0f9ee6af` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 6ac23b7d3..b3fb5565e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 22cefe04` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 0f9ee6af` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index a388b9a4e..d47c40cde 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 22cefe04` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 0f9ee6af` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index ea5db7152..8a1134793 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 22cefe04` 验证存在 +- 源码校验：以上路径均已在 `main @ 0f9ee6af` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 3f07cdceb..8740c6634 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 22cefe04`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 0f9ee6af`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 7493cbd3 \ - --to 22cefe04 \ + --from 22cefe04 \ + --to 0f9ee6af \ --repo \ --mode report-only ``` From 127962d39e1b42ff6e8a2d118d5c53539a039fce Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:52:25 +0800 Subject: [PATCH 459/465] knowledge(vllm-omni): sync ea14b04e (PR #7049) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../repos/vllm-omni/components/configuration/_index.md | 2 +- knowledge/repos/vllm-omni/components/diffusion/_index.md | 2 +- .../repos/vllm-omni/components/distributed/_index.md | 2 +- .../repos/vllm-omni/components/model-executor/_index.md | 2 +- knowledge/repos/vllm-omni/components/scheduler/_index.md | 2 +- knowledge/repos/vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/bagel/rules.md | 9 ++++++--- knowledge/repos/vllm-omni/models/catalog.md | 6 +++--- 10 files changed, 18 insertions(+), 15 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 33ab868fb..4a41bb07b 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 22cefe04b4007607ff235ea8520cd404ad44df3e + previous_audited_sha: 0f9ee6afd83c5e1dc02fe834d8d2c4c51ecd4d3b audited_ref: main - audited_sha: 0f9ee6afd83c5e1dc02fe834d8d2c4c51ecd4d3b + audited_sha: ea14b04e9f086495eb253535e6c17cf380e0fa19 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 041210b76..f43007852 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -0f9ee6afd83c5e1dc02fe834d8d2c4c51ecd4d3b`. The canonical machine baseline +ea14b04e9f086495eb253535e6c17cf380e0fa19`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 5ee1ea01c..a18e7fed2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 0f9ee6af` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ea14b04e` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 03523a97b..aa34f8fc2 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 0f9ee6af` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ ea14b04e` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 580419d60..5668f4f92 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 0f9ee6af` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ea14b04e` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b3fb5565e..b6df3af55 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 0f9ee6af` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ea14b04e` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index d47c40cde..40b2b488d 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 0f9ee6af` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ea14b04e` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 8a1134793..93341c428 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 0f9ee6af` 验证存在 +- 源码校验：以上路径均已在 `main @ ea14b04e` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/bagel/rules.md b/knowledge/repos/vllm-omni/models/bagel/rules.md index 586139de7..dfd2ca056 100644 --- a/knowledge/repos/vllm-omni/models/bagel/rules.md +++ b/knowledge/repos/vllm-omni/models/bagel/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, models, diffusion] -sources: ["PR #5775", "PR #5884", "PR #6359", "PR #7049", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/diffusion/models/lance/lance_transformer.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/models/bagel/test_step_execution.py] +sources: ["PR #5775", "PR #5884", "PR #6359", "PR #7049", vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/diffusion_engine.py, vllm_omni/diffusion/models/bagel/bagel_transformer.py, vllm_omni/diffusion/models/bagel/pipeline_bagel.py, vllm_omni/diffusion/models/lance/lance_transformer.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/test_diffusion_engine_dummy_run.py, tests/diffusion/models/bagel/test_step_execution.py] confidence: high --- @@ -75,6 +75,9 @@ cache backend；这不否定 two-stage 的 inter-stage KV transfer。 schedule points、`N - 1` 个 Euler denoise updates，所有 image path 对 `N < 2` 失败。不要把旧的 “exactly N updates”、one-step 支持或该 PR 的 benchmark 当成当前合同。 +engine dummy warmup 在 request 和 step 两种 execution mode 都必须请求至少两个 inference +steps，保证 BAGEL 至少执行一次 denoise iteration，不能把未去噪的 initial noise 当作 warmup。 + step request abort/finish 后，runner 必须先清理 request state、paged diffusion-KV 和 stale `InputBatch`，再 dispatch 后续完整 `forward()` 的 text fallback；否则先前 wave 的 tensor/KV 可 保留并污染生命周期。 @@ -82,5 +85,5 @@ step request abort/finish 后，runner 必须先清理 request state、paged dif **验收**：以两个重叠 image request 覆盖 packed index rebasing、各自 CFG/renorm、state isolation 和 output ownership；再覆盖 img2img effective geometry 不同而不合批、SP/cache rejection，以及 step request finish/abort 后的 explicit text full-forward fallback。schedule regression 必须断言 -`N < 2` 在 complete 和 step image path 均被拒绝，且不将 denoise update 数写成 `N`。^[PR #6359] -^[PR #7049] +`N < 2` 在 complete 和 step image path 均被拒绝，且不将 denoise update 数写成 `N`；dummy-run +regression 必须对 request 和 step 两种 mode 都断言两个 inference steps。^[PR #6359] ^[PR #7049] diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8740c6634..be0bbaa40 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 0f9ee6af`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ea14b04e`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 22cefe04 \ - --to 0f9ee6af \ + --from 0f9ee6af \ + --to ea14b04e \ --repo \ --mode report-only ``` From 3bb1bcf6fbcc5070b9d18c469831cf8794c148bd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 08:57:17 +0800 Subject: [PATCH 460/465] knowledge(vllm-omni): sync 348fd0fc (PR #6235) --- adapters/vllm_omni/release_baseline.yaml | 4 +- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 +- .../repos/vllm-omni/models/dots-tts/_index.md | 56 +++++++++++-------- 10 files changed, 44 insertions(+), 36 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 4a41bb07b..cb48d224c 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 0f9ee6afd83c5e1dc02fe834d8d2c4c51ecd4d3b + previous_audited_sha: ea14b04e9f086495eb253535e6c17cf380e0fa19 audited_ref: main - audited_sha: ea14b04e9f086495eb253535e6c17cf380e0fa19 + audited_sha: 348fd0fc07258a02732b07c7f32a9f3ba3b56ff9 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f43007852..9346d9148 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ea14b04e9f086495eb253535e6c17cf380e0fa19`. The canonical machine baseline +348fd0fc07258a02732b07c7f32a9f3ba3b56ff9`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index a18e7fed2..95e13337a 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ea14b04e` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 348fd0fc` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index aa34f8fc2..d0bb56301 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ea14b04e` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 348fd0fc` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 5668f4f92..1089a3b4d 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ea14b04e` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 348fd0fc` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index b6df3af55..cc81e67c5 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ea14b04e` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 348fd0fc` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 40b2b488d..04cab8602 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ea14b04e` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 348fd0fc` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 93341c428..1ef50636c 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ea14b04e` 验证存在 +- 源码校验：以上路径均已在 `main @ 348fd0fc` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index be0bbaa40..8a85ce0e9 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ea14b04e`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 348fd0fc`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 0f9ee6af \ - --to ea14b04e \ + --from ea14b04e \ + --to 348fd0fc \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/dots-tts/_index.md b/knowledge/repos/vllm-omni/models/dots-tts/_index.md index 9d7a16069..693223c13 100644 --- a/knowledge/repos/vllm-omni/models/dots-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/dots-tts/_index.md @@ -1,41 +1,55 @@ --- -title: "dots.tts" -created: 2026-09-04 +title: "dots.tts（单 stage 连续 AR 48 kHz TTS）" +created: 2026-08-13 updated: 2026-09-05 type: index tags: [vllm-omni, models, serving] -sources: ["PR #6174", vllm_omni/model_executor/models/dots_tts/, vllm_omni/deploy/dots_tts.yaml, vllm_omni/transformers_utils/configs/dots_tts.py, tests/e2e/offline_inference/test_dots_tts_expansion.py] +sources: ["PR #6174", "PR #6235", vllm_omni/model_executor/models/dots_tts/, vllm_omni/deploy/dots_tts.yaml, vllm_omni/transformers_utils/configs/dots_tts.py, vllm_omni/entrypoints/openai/tts_adapters/dots_tts.py, tests/e2e/offline_inference/test_dots_tts_expansion.py, tests/e2e/online_serving/test_dots_tts_expansion.py] confidence: high --- # dots.tts -## 正式名称与别名 +## 名称与范围 - 知识树 owner：`models/dots-tts`；上游目录 `dots_tts`。 -- pipeline registry key：`dots_tts`（`vllm_omni/config/pipeline_registry.py`）。 -- registry 登记的 architecture / stage key：`dots_tts_talker`。 +- 正式 checkpoint 为 `dots-studio/dots.tts-soar`；代码、pipeline 和 deploy 标识均为 + `dots_tts`。这是约 1.7B 的连续 AR TTS（Qwen2.5-1.5B 基座、DiT 流匹配头和 + AudioVAE），输出 48 kHz 单声道音频。^[PR #6235] +- AR registry 入口是 `DotsTTSForConditionalGeneration` → + (`dots_tts`, `dots_tts_talker`)，并有单 stage `OMNI_PIPELINES` key + `dots_tts`；模型实现位于 `model_executor/models/dots_tts/`。 -## 源码路径 +## 配置与 checkpoint 差异 -共 8 个文件： +- 默认 `dots_tts.yaml` 是单 AR stage：`max_num_seqs: 4`、`enforce_eager: true`、 + `enable_prefix_caching: false`，且默认采样上限为 4096 tokens。 +- `async_chunk: false`、全局 `dtype: bfloat16`；该初始集成没有 CUDA Graph 或 batched + side-path 优化。 -- `vllm_omni/model_executor/models/dots_tts/` -- `vllm_omni/deploy/dots_tts.yaml` -- `vllm_omni/transformers_utils/configs/dots_tts.py` +## 在线 serving 合同 -## 依赖的共享代码模块 +- PR #6235 为 `/v1/audio/speech` 注册 `DotsTTSAdapter`。adapter 只按 + `DotsTTSForConditionalGeneration` architecture 识别，`stage_keys` 为空且 + `detect_priority = 5`：不得因和 VoxCPM2 共用 `latent_generator` stage key 而把 + 请求路由给 VoxCPM2。 +- 支持 text-only 合成：`voice` 只能缺省或为 `default`；`ref_audio`、`ref_text`、 + `speaker_embedding` 和 `x_vector_only_mode` 都不支持。prompt/tokenizer 的首次构建在 + TTS executor 异步执行，避免占用 async request handler。 +- E2E 覆盖流式 WAV（含 `stream_format="audio"`）和非流式 PCM；二者都通过 + `dots_tts.yaml` 启动 L4 单卡 server。^[PR #6235] -- `vllm_omni/model_executor/models/` → [model-executor](../../components/model-executor/_index.md) -- `vllm_omni/config/stage_config/` → [configuration](../../components/configuration/_index.md) +## 共享边界 -## checkpoint、尺寸与量化 - -- `dots_tts.yaml`：dots.tts deployment config (single-stage AR talker). Conservative first-integration settings: eager execution, no CUDA Graphs, no batched side-path optimizations yet — see voxcpm2.yaml for 配置：`async_chunk: false`；`dtype: bfloat16`。 +- OpenAI 请求、TTS adapter 注册和流式音频协议属于[Serving 组件](../../components/serving/_index.md)。 +- checkpoint talker、prompt builder 和 audio side-path 属于[Model Executor 组件](../../components/model-executor/_index.md)；不要把它们的共享加载或运行时规则复制到本页。 ## 什么时候查这里 -只查 dots.tts 专有的行为、常量、注册入口和验证合同。 +- 审查 dots.tts 的 checkpoint、单 stage deploy、TTS adapter 检测，或 text-only + `/v1/audio/speech` 行为时。 +- 共享 OpenAI 协议或其他 TTS 适配器问题先进入 Serving；通用模型加载问题先进入 + Model Executor。 ## 目录内容 @@ -48,9 +62,3 @@ confidence: high 上面列出的共享模块的执行、调度、加载或 serving 合同属于 [components](../../components/_index.md)，这里只链接不复制；registry 快照和别名 清单见 [模型 catalog](../catalog.md)；一次性历史默认不落盘。 - -## 目录内容 - -| 遇到什么 | 查看哪里 | -|---|---| -| 该模型的硬门禁规则 | 尚未沉淀；由逐 commit 同步命中该 owner 时在 `rules.md` 建立 | From 77c95c15776e9c675af6dd97199f23cd517c7c03 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 09:03:33 +0800 Subject: [PATCH 461/465] knowledge(vllm-omni): sync 66d93810 (PR #6775) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../components/model-executor/rules.md | 22 ++++++++++++++++++- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- 10 files changed, 33 insertions(+), 13 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index cb48d224c..27aae5409 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: ea14b04e9f086495eb253535e6c17cf380e0fa19 + previous_audited_sha: 348fd0fc07258a02732b07c7f32a9f3ba3b56ff9 audited_ref: main - audited_sha: 348fd0fc07258a02732b07c7f32a9f3ba3b56ff9 + audited_sha: 66d938100b9bf00a34369bfbdc206b94b14d0ad8 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 9346d9148..f9f55caa4 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -348fd0fc07258a02732b07c7f32a9f3ba3b56ff9`. The canonical machine baseline +66d938100b9bf00a34369bfbdc206b94b14d0ad8`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 95e13337a..bbd747eb5 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 348fd0fc` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 66d93810` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index d0bb56301..e535522d8 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 348fd0fc` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 66d93810` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index 1089a3b4d..c893236a5 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 348fd0fc` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 66d93810` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index cc81e67c5..80d9cd39b 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 348fd0fc` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 66d93810` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 731f979e7..68244c745 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -4,7 +4,7 @@ created: 2026-07-10 updated: 2026-09-05 type: rule tags: [vllm-omni, components, model-executor] -sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149", "PR #4322", "PR #6674", "PR #6813"] +sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_runner.py, vllm_omni/platforms/interface.py, vllm_omni/platforms/musa/platform.py, vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/utils/mm_outputs.py, tests/worker/test_omni_gpu_model_runner.py, tests/worker/test_gpu_ar_model_runner.py, docs/design/feature/omni_async_output_materialization.md, vllm_omni/config/model.py, vllm_omni/config/stage_config.py, vllm_omni/config/omni_config.py, vllm_omni/engine/stage_runtime.py, vllm_omni/engine/stage_engine_startup.py, tests/engine/test_async_omni_engine_stage_init.py, vllm_omni/experimental/fullduplex/, tests/e2e/features/fullduplex/, vllm_omni/model_executor/models/common/qwen3_code_predictor.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py, vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py, tests/examples/offline_inference/test_image_task_prompts.py, tests/engine/test_arg_utils.py, "PR #3422", "PR #3642", "PR #4730", "PR #4958", "PR #5073", "PR #5074", "PR #5310", "PR #5610", "PR #5671", "PR #5777", "PR #5792", "PR #5824", "PR #5957", "PR #5976", "PR #6049", vllm_omni/engine/arg_utils.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py, vllm_omni/model_executor/models/dynin_omni/dynin_omni.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/worker/base.py, vllm_omni/worker/gpu_generation_model_runner.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py, "claude-workflow-starter-private@09dca46", "PR #4795", "PR #5842", "vllm_omni/model_executor/stage_input_processors/nemotron_voicechat.py", "vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_code2wav.py", "PR #5146", "PR #5068", "PR #5174", "PR #6076", "PR #6096", "PR #4765", "PR #5886", "PR #6061", "vllm_omni/platforms/npu/platform.py", "vllm_omni/platforms/npu/models/minimax_h3.py", "PR #5666", "PR #6152", "PR #5877", "vllm_omni/transformers_utils/configs/sensenova_u1.py", "PR #6119", "PR #6346", "PR #5452", "PR #6306", vllm_omni/worker/omni_connector_model_runner_mixin.py, vllm_omni/worker/omni_connector_validation.py, "PR #6149", "PR #4322", "PR #6674", "PR #6813", "PR #6775"] --- # Model Executor 规则 @@ -19,6 +19,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | strict stage config、known-fields/projection、service/stage 字段归属、runtime config | `strict-stage-config`：`EXEC-3a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_common_stage_config_kwargs,VllmOmniConfig.from_pipeline_config}` → 真实 startup consumer | | connector-required stage、worker/runner selection、platform worker capability | `connector-capability`：`EXEC-3c` | `engine/arg_utils.py::OmniEngineArgs.__post_init__` → `worker/omni_connector_validation.py::validate_worker_omni_connector` → selected `worker_cls.model_runner_cls` | | runner `_preprocess`、逐请求 metadata、prefill/decode phase、batch preprocess、MTP | `runner-preprocess`：本页“Runner 到模型的预处理合同” | `vllm_omni/worker/gpu_model_runner.py::{OmniGPUModelRunner._maybe_run_batch_preprocess,_preprocess,_build_model_kwargs_extra,_talker_mtp_forward}` → `vllm_omni/model_executor/models/<命中模型>` consumer | +| CUDA/NPU graph bucket padding、`seq_token_counts` 与 flat `input_ids`、codec exact-shape opt-in | `runner-preprocess`：`EXEC-4e` | GPU/NPU generation runner `_preprocess` → `maybe_unpad_input_ids` → codec model `torch.split` consumer | | stage TP/PP/DP、devices、replica、inline/subprocess、visible devices、worker 启动、容量 fail-fast | `stage-runtime`：`EXEC-3d` + 本页“Stage 并行度和设备容量必须一起验收” | `vllm_omni/config/stage_config.py::build_stage_runtime_overrides` → `vllm_omni/engine/stage_runtime.py::{StageRuntime.initialize,StageRuntime._resolve_replica_physical_devices}` → `stage_engine_startup.py::{launch_stage_replica,get_headless_replica_devices}` | | `runtime_info`、request RNG、batch compaction、跨 stage bridge/串线 | `bridge-batch`：`EXEC-1a`–`1c` | shared runner request state → stage input processor → model consumer | | cross-stage embedding width、pre-projection buffer | `bridge-batch`：`EXEC-1d` | stage HF config → `get_inputs_embeds_size()` → `GPUARModelRunner.inputs_embeds` → model projection | @@ -44,6 +45,7 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装、stage snapshot/subdir、wrapper callable 委托 | `EXEC-2a`–`EXEC-2i`，见 [loader 合同](rules-loader-contract.md) | | `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a`, `EXEC-5b` | | `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | +| `runner-preprocess` | graph-bucket padded `input_ids` 与 semantic token-count metadata | `EXEC-4e` | | `image-task-envelope` | shared image/video task example、video output consumer 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6d`，见 [image task envelope 合同](rules-image-task-envelope.md) | | `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | | `output-contract` | Omni 输出类型与字段/复制合同 | `EXEC-7a`–`EXEC-7b`，见 [输出类型合同](rules-output-contract.md) | @@ -143,6 +145,24 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru - 平台边界：查 GPU generation/AR runner 和 NPU 等平台是继承共享实现还是覆盖它；没有 live 继承或调用链证据时，不得声称所有平台合同已经一致。 - 文档与兼容：已向 out-of-tree 模型开放的字段必须从 model-contribution 入口可发现，并明确旧 runtime 和新 consumer 需要怎样配套升级；不能用某个 in-tree 模型的 fallback 充当公开合同。 +### EXEC-4e — exact-shape codec 只能取得语义长度的 `input_ids` view + +- **触发：** generation runner 的 graph bucket padding、`_preprocess`、flat `input_ids`、 + `seq_token_counts`，或 codec 模型以逐请求 token counts 分割输入发生改变。 +- **必须：** CUDA/GPU 与 Ascend NPU runner 都在 `_preprocess` 后、模型前调用同一 opt-in + helper；仅 `requires_exact_input_shape` 的模型取得截至 semantic total 的 `input_ids` + （GPU `num_tokens_unpadded` 与 NPU `scheduler_output.total_num_scheduled_tokens` 相同）。 + positions、attention metadata、graph bucket/buffer 和 forward context 继续 padded。以 + `seq_token_counts` 精确 `torch.split` flat input 的 codec consumer 必须显式 opt in。 +- **禁止：** 不得把 padding token 计入 `seq_token_counts`，不得复用 + `requires_raw_input_tokens`（只决定 token IDs 与 embeddings），不得无条件 unpad、只修 + GPU/NPU 一侧，或改写 graph capture 的 padded metadata/buffer。 +- **验收：** CPU/mock runner regression 构造 semantic 3 tokens→graph bucket 4，捕获真正进入 + model boundary 的 argument：opt-in case 断言 `input_ids.numel() == sum(seq_token_counts) == 3`， + exact bucket 不变；negative control（无 flag）仍收 4-token padded view。helper 单测覆盖 flag、 + 缺失 flag、`input_ids is None`；GPU/NPU call site 共用 helper。NPU graph hardware/E2E 须另验， + CPU mock 不证明硬件或性能 parity。^[PR #6775] + ## Async Omni output materialization ### EXEC-4d — GPU/NPU AR runner 必须在统一采样点接入 duplex hook diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 04cab8602..ce541b564 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 348fd0fc` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 66d93810` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1ef50636c..52bb0e64e 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 348fd0fc` 验证存在 +- 源码校验：以上路径均已在 `main @ 66d93810` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 8a85ce0e9..d44efb2eb 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 348fd0fc`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 66d93810`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from ea14b04e \ - --to 348fd0fc \ + --from 348fd0fc \ + --to 66d93810 \ --repo \ --mode report-only ``` From 2d5624376f96d0e4c4a1ed6ad6c2fdf07fc829b7 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 09:06:11 +0800 Subject: [PATCH 462/465] knowledge(vllm-omni): sync b6b992ac (PR #7065) --- adapters/vllm_omni/release_baseline.yaml | 4 ++-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++--- .../vllm-omni/models/higgs-audio/_index.md | 4 ++-- .../models/higgs-audio/architecture.md | 10 +++++--- .../vllm-omni/models/higgs-audio/rules.md | 23 ++++++++++++++++++- 12 files changed, 43 insertions(+), 18 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 27aae5409..0e491a5ad 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 348fd0fc07258a02732b07c7f32a9f3ba3b56ff9 + previous_audited_sha: 66d938100b9bf00a34369bfbdc206b94b14d0ad8 audited_ref: main - audited_sha: 66d938100b9bf00a34369bfbdc206b94b14d0ad8 + audited_sha: b6b992ac7baa7896c9123cea6abcd9151612a697 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index f9f55caa4..7645c083f 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -66d938100b9bf00a34369bfbdc206b94b14d0ad8`. The canonical machine baseline +b6b992ac7baa7896c9123cea6abcd9151612a697`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index bbd747eb5..8e25584b2 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ 66d93810` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ b6b992ac` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index e535522d8..3e235a58f 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ 66d93810` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ b6b992ac` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c893236a5..dfde3b78e 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ 66d93810` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ b6b992ac` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 80d9cd39b..c049bd04e 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ 66d93810` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ b6b992ac` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index ce541b564..8f8118a61 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ 66d93810` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ b6b992ac` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 52bb0e64e..1e3b1c95f 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ 66d93810` 验证存在 +- 源码校验：以上路径均已在 `main @ b6b992ac` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index d44efb2eb..19666d192 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ 66d93810`（2026-09-04）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ b6b992ac`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 348fd0fc \ - --to 66d93810 \ + --from 66d93810 \ + --to b6b992ac \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/_index.md b/knowledge/repos/vllm-omni/models/higgs-audio/_index.md index 0bda92c9e..5f1e4ec32 100644 --- a/knowledge/repos/vllm-omni/models/higgs-audio/_index.md +++ b/knowledge/repos/vllm-omni/models/higgs-audio/_index.md @@ -4,7 +4,7 @@ created: 2026-07-21 updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: ["PR #6422", vllm_omni/model_executor/models/higgs_audio_v2/, vllm_omni/model_executor/models/higgs_audio_v3/, vllm_omni/transformers_utils/configs/higgs_audio_v3.py, tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py, tests/e2e/online_serving/test_higgs_audio_v3.py] +sources: ["PR #6422", "PR #7065", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/higgs_audio_v2/, vllm_omni/model_executor/models/higgs_audio_v3/, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_tokenizer.py, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/transformers_utils/configs/higgs_audio_v3.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py, tests/e2e/online_serving/test_higgs_audio_v3.py] --- # Higgs-Audio @@ -53,7 +53,7 @@ sources: ["PR #6422", vllm_omni/model_executor/models/higgs_audio_v2/, vllm_omni | 遇到什么 | 查看哪里 | 说明 | |---|---|---| | 谱系差异表、delay-pattern 生命周期、流式窗口数学 | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | -| V3 request RNG、decode state 与 transcript test oracle | [rules](rules.md) | `HIGGS-1a` | +| V3 request RNG、decode state、voice-clone prompt sentinel 或 chunked prefill substitution | [rules](rules.md) | `HIGGS-1a`、`HIGGS-2a` | ## 配置与 checkpoint 差异（v2 vs v3 一览） diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md b/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md index c3c5e35a8..72201a85b 100644 --- a/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md +++ b/knowledge/repos/vllm-omni/models/higgs-audio/architecture.md @@ -4,7 +4,7 @@ created: 2026-07-21 updated: 2026-09-05 type: architecture tags: [vllm-omni, models] -sources: ["PR #6422", vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/model_executor/stage_input_processors/higgs_audio_v2.py, vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py, vllm_omni/deploy/README_higgs_audio_v3.md] +sources: ["PR #6422", "PR #7065", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_tokenizer.py, vllm_omni/worker/gpu_model_runner.py, vllm_omni/model_executor/stage_input_processors/higgs_audio_v2.py, vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py, vllm_omni/deploy/README_higgs_audio_v3.md] --- # Higgs-Audio 架构 @@ -35,8 +35,12 @@ sources: ["PR #6422", vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio （`build_voice_clone_prompt`,字节级一致的 system prompt）,但 `ref_audio_in_system_message`/多说话人/`profile:`/长文分块被显式拒绝 4xx;v3 用 - `<|tts|>/<|ref_text|>/<|ref_audio|>/<|text|>/<|audio|>` 模板,参考音频用 - `-100` 占位符在 prefill 换成 delay 编码后的融合嵌入。 + `<|tts|>/<|ref_text|>/<|ref_audio|>/<|text|>/<|audio|>` 模板。V3 prompt builder + 内部以 `-100` 表示参考音频 embedding 槽；进入 vLLM 前 tokenizer adapter 将它们 + 换成非 audio-continuation 的 `<|tts|>` 词表 ID，并把绝对槽位置随 reference codes + 一起传递。talker 在 transformer layers 前按这些位置（包括 chunked prefill 的局部 + span）注入 delay 编码后的融合 embedding；绕过 engine validation 的旧内部调用仍可 + 使用 `-100` fallback。具体必须执行的合同见 [HIGGS-2a](rules.md)。 - v3 config 从 checkpoint tokenizer 解析特殊 token 与 eos（缺失即 raise）。 - 采样注意：v2 YAML 注释记载 greedy 会塌缩 codebook argmax，因此 v2 profile 启用 sampling； #6422 不改变也不重新证明 v2。V3 则显式消费 vLLM 的 per-request `torch.Generator`，将一个 diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/rules.md b/knowledge/repos/vllm-omni/models/higgs-audio/rules.md index 4198da583..211e087b5 100644 --- a/knowledge/repos/vllm-omni/models/higgs-audio/rules.md +++ b/knowledge/repos/vllm-omni/models/higgs-audio/rules.md @@ -4,7 +4,7 @@ created: 2026-09-05 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #6422", vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/worker/gpu_model_runner.py, tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py, tests/e2e/online_serving/test_higgs_audio_v3.py] +sources: ["PR #6422", "PR #7065", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_tokenizer.py, vllm_omni/worker/gpu_model_runner.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py, tests/e2e/online_serving/test_higgs_audio_v3.py] confidence: high --- @@ -26,3 +26,24 @@ confidence: high reuse 与 same-ID replacement，并验证 runner 把 live `input_batch.req_ids` 传入 decode metadata。 ROCm 配置测试只证明 V3 Stage 0 的 `TRITON_ATTN` overlay/collection；作者报告的单次 MI300X case 不构成通用确定性、音质、吞吐或跨平台支持。^[PR #6422] + +## HIGGS-2a — voice-clone 占位语义必须在词表校验边界外显并保持位置对应 + +- 触发：修改 Higgs-Audio V3 的 voice-clone prompt、`ref_audio` metadata、engine token + validation、prefill/chunked-prefill embedding substitution，或 offline/online prompt producer。 +- 必须：仅在模型本地 `build_prompt` 中保留负值 sentinel；每个提交给 vLLM 的 token ID 必须 + 属于 tokenizer vocabulary。adapter 必须同时把 sentinel 替换为不触发 audio-continuation + sampler 状态的有效 filler（当前为 `<|tts|>`），并记录按 prompt 顺序排列的绝对 + `audio_placeholder_positions`。online serving 与 offline example 都必须传递这些位置和 + delay-pattern reference codes；talker 必须在 transformer layers 前按每个 request span 的 + absolute positions 选择对应 code rows，因而 full 与 chunked prefill 都能注入正确 embedding。 + 仅供绕过 engine validation 的 legacy internal caller 保留 `-100` substitution fallback。 +- 禁止：把负 sentinel 直接送过共享 token validation；用 `<|audio|>` 或任何会触发采样 + continuation 语义的 filler；按当前 chunk 的局部索引、token identity 或未对齐的 code row + 猜测参考音频位置；只修 online serving 而遗漏 offline producer；删除 legacy fallback 却未 + 迁移其 internal caller。 +- 验收：prompt smoke 断言 engine IDs 全部非负、每个 placeholder filler 为 `tts_id` 且不为 + `audio_id`，并保留 position metadata；单元测试分别证明完整 prefill、只含 reference span + 一部分的 chunked prefill 和 legacy `-100` caller 注入正确 code rows；serving smoke 证明相同 + 路径重写时 cache salt 仍变化。H100 voice-clone E2E 未在该 PR 的 lanes 运行，不能把 unit + coverage 宣称为 end-to-end 证据。^[PR #7065] From 56da7e4c6d4713a6749eb3cb6727af673d227f42 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 09:09:45 +0800 Subject: [PATCH 463/465] knowledge(vllm-omni): sync ae57e406 (PR #5675) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- knowledge/repos/vllm-omni/ci/_index.md | 1 + .../ci/rules-whisper-transcription.md | 31 +++++++++++++++++++ .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- 11 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 knowledge/repos/vllm-omni/ci/rules-whisper-transcription.md diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 0e491a5ad..8a5ef34f7 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: 66d938100b9bf00a34369bfbdc206b94b14d0ad8 + previous_audited_sha: b6b992ac7baa7896c9123cea6abcd9151612a697 audited_ref: main - audited_sha: b6b992ac7baa7896c9123cea6abcd9151612a697 + audited_sha: ae57e406fd02516ef5ee8cd40a1458328e13a2a5 # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 7645c083f..654578b3c 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -b6b992ac7baa7896c9123cea6abcd9151612a697`. The canonical machine baseline +ae57e406fd02516ef5ee8cd40a1458328e13a2a5`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/ci/_index.md b/knowledge/repos/vllm-omni/ci/_index.md index d9133ff30..e1ed7212b 100644 --- a/knowledge/repos/vllm-omni/ci/_index.md +++ b/knowledge/repos/vllm-omni/ci/_index.md @@ -211,5 +211,6 @@ sources: [.buildkite/cuda/pipeline.yml, docs/contributing/ci/test_system_overvie | 审查硬件 lane（含 MiniMax-H3 DLO DP2 ready smoke 的证据边界）、回归 fence、CI 工具供应链、ASR 文本比较或 xdist/shared fixture | [CI rules](rules.md) | | AMD/ROCm timeout、quarantine、memory signal、Qwen3-TTS argv 或 Qwen3-Omni control-plane fixture | [AMD/ROCm CI rules](rules-amd.md) | | CUDA L4 Kubernetes preset、GPU-count shard、resource/retry policy | [L4 Kubernetes CI rules](rules-l4-k8s.md) | +| Whisper 转写 helper 的 GPU 首选、16 GiB 门槛、CPU fallback 或 CUDA ready/merge source dependency | [Whisper 转写 CI 规则](rules-whisper-transcription.md) | | 查看仓库特有 CI 陷阱 | [CI guides](guides/_index.md) | | 调查历史 CI 失败 | [CI incidents](incidents/_index.md) | diff --git a/knowledge/repos/vllm-omni/ci/rules-whisper-transcription.md b/knowledge/repos/vllm-omni/ci/rules-whisper-transcription.md new file mode 100644 index 000000000..8e5f6cdd2 --- /dev/null +++ b/knowledge/repos/vllm-omni/ci/rules-whisper-transcription.md @@ -0,0 +1,31 @@ +--- +title: "vLLM-Omni Whisper 转写 CI 规则" +created: 2026-09-05 +updated: 2026-09-05 +type: rule +tags: [vllm-omni, ci] +sources: ["PR #5675", .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, tests/helpers/media.py, tests/helpers/tests/test_media.py] +confidence: high +--- + +# Whisper 转写 CI 规则 + +这是 [CI 入口](_index.md) 下的专项规则；Qwen3-TTS ready/merge 的其余收集边界见 +[TTS CI 规则](rules-tts.md)。 + +## OMNI-CI-3e — Whisper 转写设备选择必须以容量首个命中和对应 lane 路由共同约束 + +- 触发：修改 `tests/helpers/media.py` 的 Whisper 转写设备选择、显存门槛或 CPU fallback，或修改其 + CUDA ready/merge `source_file_dependencies`。 +- 必须：从最高可见设备索引向下探测，并在第一个可用显存不少于 16 GiB 的设备立即停止、设为 + Whisper 设备；无设备满足时回退 CPU。保留该门槛保护单卡 server-plus-Whisper OOM 的说明；单元测试 + 必须覆盖等于门槛、低 1 byte、最高索引优先、遇到不足设备后的继续探测、全不足回退，以及 probe + 顺序/early break。凡调用该 helper 的 CUDA TTS lane，ready/merge 的 source dependency 都必须显式 + 包含 `tests/helpers/media.py`。 +- 禁止：恢复无条件的 `n > 1` 选择；为寻找 most-free 设备扫描全部卡；把 16 GiB 当作已预留显存；或 + 让 helper-only diff 绕过其调用的 CUDA TTS gate。 +- 验收：`tests/helpers/tests/test_media.py::test_select_whisper_device_by_available_memory` 覆盖上述边界和 + probe order；`.buildkite/cuda/test-ready.yml` 的 Qwen3-TTS CustomVoice、以及 + `.buildkite/cuda/test-merge.yml` 的 Qwen3-TTS CustomVoice 与 Base，均将 + `tests/helpers/media.py` 列为 source dependency。与 TTS ready/merge oracle 的其余收集边界同时遵守 + [OMNI-CI-1d](rules-tts.md#omni-ci-1d-qwen3-tts-base-的-dummy-ready-oracle-与-real-weight-merge-coverage-必须分离)。^[PR #5675] diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 8e25584b2..225765693 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ b6b992ac` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ ae57e406` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 3e235a58f..7f01cb788 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ b6b992ac` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ ae57e406` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index dfde3b78e..c4b4533a7 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ b6b992ac` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ ae57e406` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index c049bd04e..8efe6f301 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ b6b992ac` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ ae57e406` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index 8f8118a61..c58f5b833 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ b6b992ac` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ ae57e406` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 1e3b1c95f..153432359 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ b6b992ac` 验证存在 +- 源码校验：以上路径均已在 `main @ ae57e406` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 19666d192..9176e77be 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ b6b992ac`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ ae57e406`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from 66d93810 \ - --to b6b992ac \ + --from b6b992ac \ + --to ae57e406 \ --repo \ --mode report-only ``` From 0c7663c4095c8c7575e1bd42c0a179ea0e0258fa Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 09:12:55 +0800 Subject: [PATCH 464/465] knowledge(vllm-omni): sync 1e74807c (PR #4982) --- adapters/vllm_omni/release_baseline.yaml | 4 +-- doc/architecture/KNOWLEDGE.md | 2 +- .../components/configuration/_index.md | 2 +- .../vllm-omni/components/diffusion/_index.md | 2 +- .../components/distributed/_index.md | 2 +- .../components/model-executor/_index.md | 2 +- .../vllm-omni/components/scheduler/_index.md | 2 +- .../vllm-omni/components/serving/_index.md | 2 +- knowledge/repos/vllm-omni/models/catalog.md | 6 ++-- .../repos/vllm-omni/models/moss-tts/_index.md | 3 +- .../repos/vllm-omni/models/moss-tts/rules.md | 33 ++++++++++++++++++- 11 files changed, 46 insertions(+), 14 deletions(-) diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 8a5ef34f7..717d0e566 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -2,9 +2,9 @@ schema_version: 1 upstream: repository: vllm-project/vllm-omni - previous_audited_sha: b6b992ac7baa7896c9123cea6abcd9151612a697 + previous_audited_sha: ae57e406fd02516ef5ee8cd40a1458328e13a2a5 audited_ref: main - audited_sha: ae57e406fd02516ef5ee8cd40a1458328e13a2a5 + audited_sha: 1e74807c356be5cba4239faae7c38763fe1a44be # These fingerprints are generated from sorted registry/deploy inventories. # They make the baseline exact without duplicating hundreds of entries here. diff --git a/doc/architecture/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md index 654578b3c..1ee4ea588 100644 --- a/doc/architecture/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -42,7 +42,7 @@ extensions merely because they are absent from the common source. history was removed — used for page frontmatter). - **Code-mirror pin:** the `knowledge/repos/vllm-omni/components/` source maps are verified against vllm-omni `main @ -ae57e406fd02516ef5ee8cd40a1458328e13a2a5`. The canonical machine baseline +1e74807c356be5cba4239faae7c38763fe1a44be`. The canonical machine baseline is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against diff --git a/knowledge/repos/vllm-omni/components/configuration/_index.md b/knowledge/repos/vllm-omni/components/configuration/_index.md index 225765693..12ec26f92 100644 --- a/knowledge/repos/vllm-omni/components/configuration/_index.md +++ b/knowledge/repos/vllm-omni/components/configuration/_index.md @@ -15,7 +15,7 @@ sources: ["claude-workflow-starter-private@296ea45", vllm_omni/config/] `tests/diffusion/test_diffusion_config_propagation.py`，以及各公开入口附近的配置测试 - 部署配置：`vllm_omni/deploy/*.yaml`，以及 `pipeline_registry.py`、 `endpoint_policy.py`、`server_settings.py`、`yaml_util.py`、`composable_parallel/` -- 源码校验：以上路径在 `main @ ae57e406` 验证存在；机器基线见 +- 源码校验：以上路径在 `main @ 1e74807c` 验证存在；机器基线见 `adapters/vllm_omni/release_baseline.yaml` ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/components/diffusion/_index.md b/knowledge/repos/vllm-omni/components/diffusion/_index.md index 7f01cb788..143d425a7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/_index.md +++ b/knowledge/repos/vllm-omni/components/diffusion/_index.md @@ -10,7 +10,7 @@ sources: ["PR #5543", vllm_omni/platforms/cuda/platform.py, vllm_omni/diffusion/ # Diffusion - 源码入口：`vllm_omni/diffusion/` 全树，含 16 个子模块：attention、cache、distributed、executor、hooks、layers、lora、model_loader、models、offloader、postprocess、profiler、quantization、sched、utils、worker -- 源码校验：以上子模块均已在 `main @ ae57e406` 验证存在；MiniMax-H3 的 VAE eager +- 源码校验：以上子模块均已在 `main @ 1e74807c` 验证存在；MiniMax-H3 的 VAE eager ops 仍由模型 owner 维护，其他 shared/模型机制按各自规则审查 - 主要职责：多个 diffusion 模型共用的 pipeline、执行循环、scheduler 接入和运行机制 diff --git a/knowledge/repos/vllm-omni/components/distributed/_index.md b/knowledge/repos/vllm-omni/components/distributed/_index.md index c4b4533a7..710f03ce4 100644 --- a/knowledge/repos/vllm-omni/components/distributed/_index.md +++ b/knowledge/repos/vllm-omni/components/distributed/_index.md @@ -14,7 +14,7 @@ sources: [vllm_omni/distributed/omni_connectors/, vllm_omni/distributed/omni_coo `vllm_omni/distributed/omni_coordinator/`（协调器与 load balancer） - 知识面另覆盖跨 stage ZMQ 路由/端口分配（`vllm_omni/engine/stage_engine_startup.py::OmniMasterServer`） ——组件划分服务知识归属，与 manifest 运行时粒度不同 -- 源码校验：以上路径与下列锚点均已在 `main @ ae57e406` 验证存在： +- 源码校验：以上路径与下列锚点均已在 `main @ 1e74807c` 验证存在： `OmniConnectorBase`（connectors/base.py:12）、`OmniKVTransferManager` （kv_transfer_manager.py:341）、`LoadBalancer` 三实现（load_balancer.py:39/64/74/102）、 `OmniMasterServer._allocate_route_locked`（stage_engine_startup.py:254） diff --git a/knowledge/repos/vllm-omni/components/model-executor/_index.md b/knowledge/repos/vllm-omni/components/model-executor/_index.md index 8efe6f301..08b40fcd4 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/_index.md +++ b/knowledge/repos/vllm-omni/components/model-executor/_index.md @@ -13,7 +13,7 @@ sources: [] `vllm_omni/model_extras/`（模型专有请求/prompt 转换）、`vllm_omni/worker/`（gpu_*_worker、 gpu_*_model_runner、mixins）、`vllm_omni/inputs/`（runner 输入预处理）和设备平台层 `vllm_omni/platforms//` -- 源码校验：以上路径均已在 `main @ ae57e406` 验证存在；stage 配置已经迁移到 +- 源码校验：以上路径均已在 `main @ 1e74807c` 验证存在；stage 配置已经迁移到 `vllm_omni/deploy/`，NPU/XPU 等平台可继续拥有自己的 worker 覆盖 - 测试入口：共享 runner 行为看 `tests/worker/`，具体模型 consumer 看 `tests/model_executor/` - 主要职责：AR/LLM stage、stage 配置、并行与设备启动、runner 到模型的输入预处理合同和跨阶段数据桥接 diff --git a/knowledge/repos/vllm-omni/components/scheduler/_index.md b/knowledge/repos/vllm-omni/components/scheduler/_index.md index c58f5b833..b8724dd29 100644 --- a/knowledge/repos/vllm-omni/components/scheduler/_index.md +++ b/knowledge/repos/vllm-omni/components/scheduler/_index.md @@ -11,7 +11,7 @@ sources: [vllm_omni/core/sched/omni_ar_scheduler.py, vllm_omni/core/sched/omni_g - 源码入口：`vllm_omni/core/sched/`（`omni_ar_scheduler.py`、`omni_generation_scheduler.py`、 `omni_scheduler_mixin.py`、`omni_scheduling_coordinator.py`）和 `vllm_omni/core/prefix_cache.py` -- 源码校验：以上路径与下列类均已在 `main @ ae57e406` 验证存在：`OmniARScheduler`（:73）、 +- 源码校验：以上路径与下列类均已在 `main @ 1e74807c` 验证存在：`OmniARScheduler`（:73）、 `OmniARAsyncScheduler`（:815）、`OmniGenerationScheduler`（:29）、 `OmniSchedulerMixin`（:64）、`OmniTensorPrefixCache`（prefix_cache.py:33） - 官方 draft source map：`docs/design/module/ar_runtime.md`；旧继承关系、请求流转图只在 diff --git a/knowledge/repos/vllm-omni/components/serving/_index.md b/knowledge/repos/vllm-omni/components/serving/_index.md index 153432359..647b96ced 100644 --- a/knowledge/repos/vllm-omni/components/serving/_index.md +++ b/knowledge/repos/vllm-omni/components/serving/_index.md @@ -10,7 +10,7 @@ sources: [] # Serving - 主要源码入口：`vllm_omni/entrypoints/`（cli、openai、openpi 及 omni/async_omni 入口）和 `vllm_omni/engine/`（orchestrator、stage engine core、stage pool/runtime、output processor） -- 源码校验：以上路径均已在 `main @ ae57e406` 验证存在 +- 源码校验：以上路径均已在 `main @ 1e74807c` 验证存在 - 主要职责：用户入口、请求解析、在线服务和 engine 边界 ## 什么时候查这里 diff --git a/knowledge/repos/vllm-omni/models/catalog.md b/knowledge/repos/vllm-omni/models/catalog.md index 9176e77be..5e567bc80 100644 --- a/knowledge/repos/vllm-omni/models/catalog.md +++ b/knowledge/repos/vllm-omni/models/catalog.md @@ -10,7 +10,7 @@ sources: [vllm_omni/model_executor/models/registry.py, vllm_omni/diffusion/regis # 模型代码入口与 registry 快照 本页提供模型描述到代码目录的自动定位入口，不维护逐模型 class 映射。下方计数是 -`main @ ae57e406`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 +`main @ 1e74807c`（2026-09-05）快照，数字会漂移，不能凭它断言“不支持”。 ## Direct 模型代码入口 @@ -137,8 +137,8 @@ step_audio_2_asr、voxcpm2、voxtral_tts、wan2_2_ti2v ```bash python tools/audit_vllm_omni_release.py \ - --from b6b992ac \ - --to ae57e406 \ + --from ae57e406 \ + --to 1e74807c \ --repo \ --mode report-only ``` diff --git a/knowledge/repos/vllm-omni/models/moss-tts/_index.md b/knowledge/repos/vllm-omni/models/moss-tts/_index.md index baa138c25..7ae1503ff 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/_index.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/_index.md @@ -4,7 +4,7 @@ created: 2026-07-21 updated: 2026-09-05 type: index tags: [vllm-omni, models] -sources: ["PR #5635", "PR #6664", "PR #6543", vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py, vllm_omni/model_executor/models/moss_tts/, vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py, vllm_omni/model_executor/models/moss_tts_nano/, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/] +sources: ["PR #5635", "PR #6664", "PR #6543", "PR #4982", vllm_omni/model_executor/models/registry.py, vllm_omni/config/pipeline_registry.py, vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/moss_tts/, vllm_omni/model_executor/models/moss_tts/reference_encoder.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py, vllm_omni/model_executor/models/moss_tts_nano/, vllm_omni/model_executor/stage_input_processors/moss_tts.py, vllm_omni/deploy/] --- # MOSS-TTS 家族 @@ -57,6 +57,7 @@ sources: ["PR #5635", "PR #6664", "PR #6543", vllm_omni/model_executor/models/re | delay 生命周期、伪文本 logits、双代 tokenizer | [architecture](architecture.md) | 数据流与 reviewer 陷阱 | | codec v1/v2 选择、projection/checkpoint topology | [rules](rules.md#direct-代码快速入口) | loader 门禁与测试缺口 | | online request `seed`、Nano/Local/Delay/Realtime 的可复现性或并发 | 同页 `MOSSTTS-3a` | adapter precedence 与各变体 RNG 边界 | +| MOSS reference audio 的 cache、并发 encode、TTSD 双 speaker 或 prefix salt | 同页 `MOSSTTS-4a` | content/slot/flight identity、批处理和失败隔离 | ## 配置与 checkpoint 差异（8 份 deploy 速查） diff --git a/knowledge/repos/vllm-omni/models/moss-tts/rules.md b/knowledge/repos/vllm-omni/models/moss-tts/rules.md index ea0c58338..a7db1386a 100644 --- a/knowledge/repos/vllm-omni/models/moss-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/moss-tts/rules.md @@ -4,7 +4,7 @@ created: 2026-09-02 updated: 2026-09-05 type: rule tags: [vllm-omni, models, model-executor] -sources: ["PR #5635", "PR #6908", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, "PR #6241", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py", "PR #6543", vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, tests/entrypoints/openai_api/test_tts_adapter.py] +sources: ["PR #5635", "PR #6908", vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py, vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py, "PR #6241", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py", "vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py", "PR #6543", "PR #4982", vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/moss_tts/reference_encoder.py, vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py, tests/entrypoints/openai_api/test_tts_adapter.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/model_executor/models/moss_tts/test_reference_encoder.py] confidence: high --- @@ -20,6 +20,7 @@ confidence: high | codec-v2 NPU RoPE、`npu_rotary_mul`、GPT-J pairs | `MOSSTTS-1b` | `audio_tokenizer_v2.py::apply_rope` | | `_ProjectedTransformer`、`in_proj`/`out_proj`、Identity | `MOSSTTS-1a` | `audio_tokenizer.py::_ProjectedTransformer` → codec checkpoint parameter names | | online request `seed`、adapter additional information、Nano/Local/Delay/Realtime reproducibility | `MOSSTTS-3a` | `tts_adapters/moss_tts.py::_MossTTSAdapterBase.build` → variant model seed/RNG consumer | +| reference audio cache、single-flight、micro-batch、TTSD 两 speaker | `MOSSTTS-4a` | `reference_encoder.py::MossReferenceEncoder` → `serving_speech.py::_encode_moss_references` | ## MOSSTTS-1a — codec 代际与 module topology 必须由 checkpoint 原始结构决定 @@ -80,3 +81,33 @@ confidence: high 覆盖 stage default，且 request 缺省时回退 default；模型级结论分别验证 consumer。Nano 的 并发异 seed 隔离需要在 shipping `max_num_seqs=4` 下单独以 per-request generator 验证，不能 由 adapter CPU contract 或单请求结果推出。^[PR #6543] + +## MOSSTTS-4a — reference encoder 必须保持内容、slot 与 flight identity + +- 触发：修改 MOSS `reference_encoder.py`、reference-audio speaker/code cache、 + `serving_speech.py` 的 MOSS reference handoff，或 reference preprocessing/batching。 +- 强制：server 只惰性构造一个 `MossReferenceEncoder`；在 model package 从 processor 推导 + `n_vq` 和 working sample rate，Local reference encode 固定为 24 kHz，而不是其 48 kHz + output rate。anonymous clip 必须 resolve 后才选 speaker-code cache key，优先使用 resolved + waveform artifact key、缺失时回退 resolve identity，并以 MOSS variant 与 `n_vq` namespace。 + cache 内只在 code 值可容纳时 compact 为 int32；每个 caller 都得到不 alias cache 的独立 int64 + tensor。 +- 强制：named key 只能是 normalized uploaded `voice_name` 加 positive `created_at`，并且只用于 + 该 uploaded voice 自己的 reference；placeholder/unregistered/inline voice 必须 content-addressed。 + TTSD 的 slot 0 才可用 named key，slot 1 必须 anonymous；`ref_audio_cache_key` 与 + `ref_audio_2_cache_key` 必须按 slot 保存，不能按 concurrent completion order 保存。 +- 强制：同一 cold reference 的 concurrent miss 必须 join 一个 shielded flight；flight slot 只能由 + task completion callback 以 identity guard 回收，不能因 creator waiter cancel/return 而撤销。 + cold encode 在有界 coalescing window/max batch 内合批；每 clip prep failure 独立，batch forward + failure 必须回退为逐项 encode，不能让一个坏 clip poison 同批请求。 +- 禁止：两个 TTSD reference 都按 request voice key；将 raw locator 当成跨 locator content + identity；cache key 与 flight key 对 voice case 使用不同 normalization；向 caller 返回 cache + storage；串行 encode TTSD 两 reference；或在 creator exit 时 cleanup flight。 +- 验收：CPU tests 覆盖相同 content 的不同 locator 只 encode 一次、re-upload 的 `created_at` 只 + invalidate named voice、default/inline voice 不 collide、caller mutation 不改 cache，以及 out-of- + range code 保持 int64。并发 tests 覆盖同 ref 只 resolve/encode 一次、creator cancellation 后 + later waiter 仍 join 已有 flight、distinct refs 在 bounded batch 内执行、prep failure 与 batch + fallback 只影响失败 clip。TTSD test 必须用 uploaded voice 加不同双 reference，断言 codes 不同、 + 两个 cache salt 保持 slot order，并证明二者进入同一 batch window。PR 的 A/B 只支持此 + preprocessing 的 content-addressed/single-flight/micro-batch/TTSD pair-encode 性能结论；同路径 + warm-hit parity 属于既有 speaker cache，不能归因于本规则。^[PR #4982] From 3dc84a2631b0e0445318fbef35f7d5654663b15a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 5 Sep 2026 09:36:13 +0800 Subject: [PATCH 465/465] fix(knowledge): restore Direct routing conformance --- .../components/configuration/rules.md | 16 ---------- .../vllm-omni/components/diffusion/rules.md | 24 +++++++------- .../components/model-executor/rules.md | 32 +++++++++---------- .../repos/vllm-omni/models/bagel/rules.md | 6 ++++ .../vllm-omni/models/boogu-image/rules.md | 6 ++++ .../vllm-omni/models/cosyvoice3/rules.md | 6 ++++ .../repos/vllm-omni/models/dots-tts/rules.md | 6 ++++ .../vllm-omni/models/flux2-klein/rules.md | 6 ++++ .../models/hidream-o1-image/rules.md | 6 ++++ .../vllm-omni/models/higgs-audio/rules.md | 7 ++++ .../vllm-omni/models/indextts-2-5/rules.md | 6 ++++ .../vllm-omni/models/longcat-image/rules.md | 6 ++++ .../repos/vllm-omni/models/ltx2/rules.md | 13 ++++++-- .../repos/vllm-omni/models/mage-vl/rules.md | 6 ++++ .../vllm-omni/models/mammoth-moda2/rules.md | 7 ++++ .../vllm-omni/models/minicpm-o-4-5/rules.md | 14 ++++++-- .../vllm-omni/models/minimax-h3/rules.md | 11 +++++++ .../vllm-omni/models/minimax-music3/rules.md | 6 ++++ .../models/nemotron-voicechat/rules.md | 6 ++++ .../vllm-omni/models/personaplex/rules.md | 8 +++++ knowledge/repos/vllm-omni/models/pi0/rules.md | 6 ++++ .../vllm-omni/models/qwen-image/rules.md | 8 ++++- .../repos/vllm-omni/models/qwen-omni/rules.md | 6 ++++ .../vllm-omni/models/sana-video/rules.md | 7 ++++ .../vllm-omni/models/step-audio2/rules.md | 7 ++++ .../vllm-omni/models/voxtral-tts/rules.md | 6 ++++ test/test_v2_p0.py | 3 +- 27 files changed, 189 insertions(+), 52 deletions(-) diff --git a/knowledge/repos/vllm-omni/components/configuration/rules.md b/knowledge/repos/vllm-omni/components/configuration/rules.md index ec45a0fe6..18fdaa04e 100644 --- a/knowledge/repos/vllm-omni/components/configuration/rules.md +++ b/knowledge/repos/vllm-omni/components/configuration/rules.md @@ -22,7 +22,6 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR |---|---|---| | strict schema、unknown field、alias、flat→nested、structured/legacy/direct parity、typed projection | `strict-normalization`：`VOMNI-CFG-1a`–`1h` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,strip_parent_engine_args}` → `vllm_omni/config/omni_config.py::{_build_diffusion_config_projection,VllmOmniConfig.from_pipeline_config}` → `vllm_omni/engine/stage_init_utils.py::{build_engine_args_dict,build_engine_args_dict_from_omni_stage_config}` | | deploy YAML、`base_config`、pipeline/stage overlay、pipeline-owned alias/hook、headless/offline parity、最终逐 stage config | `deploy-topology`：`CONF-3a`, `CONF-4b`, `CONF-5a`, [CONF-5i](rules-topology-profiles.md#conf-5i-pipeline-专属-alias-与-hook-必须在通用配置层声明归一并消费), [CONF-5j](rules-topology-profiles.md#conf-5j-pipelineconfig-在构造时必须验证终端-topology) | `vllm_omni/config/stage_config.py::{resolve_deploy_yaml,load_deploy_config,normalize_pipeline_cli_overrides,merge_pipeline_deploy,build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/config_factory.py::StageConfigFactory.create_from_model` → `vllm_omni/engine/stage_init_utils.py::_resolve_model_path` | -| MiniCPM-o shipping profile、duplex/chat 共服、Talker sampling floor | model-specific [`MCPMO-4e`](../../models/minicpm-o-4-5/rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → `merge_pipeline_deploy`；duplex adapter runtime sampling override | | composable strategy、axis、routing、load balancing、`strategy-config` | `composable-strategy`：`CONF-4a` | `vllm_omni/config/composable_parallel/strategy_loader.py::{parse_strategy_specs,load_strategy_specs}` → `translator.py::translate_strategy_stack` → `apply.py::apply_strategy_specs` → `config_factory.py::{StageConfigFactory._apply_strategy_specs,StageConfigFactory._reconcile_strategy_with_cli}` | | `gpu_memory_utilization`、`kv_cache_memory_bytes`、多 stage 共卡、小显存 OOM | `deploy-memory`：`CONF-1a`, `CONF-2a` | `vllm_omni/config/stage_config.py::{build_stage_runtime_overrides,_build_engine_args}` → `vllm_omni/config/omni_config.py::{_build_runtime_config,_build_parallel_config,VllmOmniConfig.from_pipeline_config}` | | multi-stage HF sub-config、stage quantization view、`hf_config_name` | `stage-model-config`：`VOMNI-CFG-1i` | pipeline stage declaration → `OmniModelConfig::{draw_hf_text_config,get_model_arch_config}` → vLLM quantization selection | @@ -30,21 +29,6 @@ sources: ["claude-workflow-starter-private@296ea45", "PR #4281", "PR #5031", "PR | stage transport、`requires_full_payload_input`、topology projection 或 override rejection | `stage-transport`：`VOMNI-CFG-1p` | `stage_config.py::{StagePipelineConfig,_build_engine_args}` → `omni_config.py::_build_model_config` → `engine/arg_utils.py::OmniEngineArgs` | | SymmMem Ulysses transport | [VOMNI-CFG-1q](rules-diffusion-parallel-transport.md) + `DIFF-4x` | deploy → parallel config → Ulysses | | HF cache snapshot path、空 `config.json`、name-based pipeline fallback | [`model-reference-routing`](rules-model-reference-routing.md#conf-7a-模型引用解析必须物化对象存储配置并只从受控名称组件匹配)：`CONF-7a` | `config_factory.py::{_name_match_candidate,StageConfigFactory._try_infer_model_type}` → `pipeline_registry.py::OMNI_PIPELINES` → `tests/config/test_config_factory.py::TestNameMatchCandidateSnapshotPaths` | -| Qwen3-TTS adaptive codec chunk keys、fixed-ramp precedence | [Qwen3-TTS `Q3TTS-3e`](../../models/qwen3-tts/rules.md#q3tts-3e-adaptive-ramp-是-host-side每段-opt-in-控制器不是-cuda-graph-计划) | `deploy/qwen3_tts{,_high_concurrency}.yaml` connector `extra` → `stage_input_processors/chunk_size_utils.py::parse_adaptive_config` | - -| 审查组 | 什么时候触发 | 规则 ID | -|---|---|---| -| `core` | 每次配置审查 | `VOMNI-CFG-1b`, `VOMNI-CFG-1c` | -| `strict-normalization` | schema、alias、unknown field、structured/legacy/direct 路径 | `VOMNI-CFG-1a`, `VOMNI-CFG-1b`, `VOMNI-CFG-1c`, `VOMNI-CFG-1d`, `VOMNI-CFG-1e`, `VOMNI-CFG-1f`, `VOMNI-CFG-1g`, `VOMNI-CFG-1h` | -| `deploy-memory` | 显存预算、KV pin、多 stage 共卡 | `CONF-1a`, `CONF-2a` | -| `deploy-topology` | deploy、overlay、headless、topology wiring | `CONF-3a`–`3d`；`CONF-4b` 见 [并行拓扑合同](rules-parallel-topology.md)；`CONF-5a`–`5j` 见 [topology 与部署 profile](rules-topology-profiles.md)；MiniCPM-o profile 见 `MCPMO-4e` | -| `composable-strategy` | strategy axis、routing、load balancing | `CONF-4a`, `CONF-4c`–`4d`，见 [并行拓扑合同](rules-parallel-topology.md) | -| `stage-model-config` | HF nested config、stage-specific quantization/text config | `VOMNI-CFG-1i` | -| `stage-config-propagation` | pipeline sampling constraints、StageConfig serialization/runtime config | `VOMNI-CFG-1o` | -| `stage-transport` | full-payload transport capability、topology projection、deploy/CLI override | `VOMNI-CFG-1p`，见 [stage transport capability](rules-stage-transport.md) | -| `diffusion-parallel-transport` | SymmMem Ulysses flag | [VOMNI-CFG-1q](rules-diffusion-parallel-transport.md) + `DIFF-4x` | -| `model-reference-routing` | object-storage materialization、HF cache snapshot、name fallback | `CONF-7a`，见 [模型引用路由规则](rules-model-reference-routing.md) | -| `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `VOMNI-CFG-0a`, `VOMNI-CFG-0b` | ## 配置归一化与新老路径一致性 diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules.md b/knowledge/repos/vllm-omni/components/diffusion/rules.md index c0391ac06..5eb7e95d7 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules.md @@ -15,6 +15,19 @@ confidence: high ## Direct 代码快速入口 +| PR 描述信号 | 规则入口 | +|---|---| +| graph/eager、compile、solver、RNG、dtype/device | `DIFF-1a`–`DIFF-1j` | +| checkpoint remap、HWR、HSDP、offload、quantization | `DIFF-2a`–`DIFF-2z`；[checkpoint/loading](rules-checkpoint-loading.md) | +| component、multi-DiT、LoRA、request cache lifecycle | [component lifecycle](rules-component-lifecycle.md)、[LoRA](rules-lora.md) | +| quality evidence、distributed VAE | `DIFF-3a`、`DIFF-3b` | +| paged KV、attention backend/parallel、platform、admission | `DIFF-4a`–`DIFF-4x`；[system runtime](rules-system-runtime.md) | +| worker reply、shutdown、final output/formatter | `DIFF-1s`、`DIFF-1aa`、`DIFF-4z`；[output/runtime](rules-output-lifecycle.md) | +| service/forward/step metrics | `DIFF-5a`、`DIFF-5b`；[metrics](rules-metrics.md) | +| worker title/rank observability | `DIFF-4w`；[worker observability](rules-worker-observability.md) | + +## 完整代码路由 + - **DIFF-0a — PR 描述先选共享执行地图。** 按 title/body 的 graph、RNG、checkpoint、HSDP、quantization 或质量信号命中下表，再用 pinned files 验证 owner。 - **DIFF-0b — 共享 runner 与模型 pipeline 分层。** 先查命中行的共享 producer；只有 live 调用链落到单模型 pipeline/checkpoint/常量时才进入模型 owner，不横扫其他模型。 @@ -39,17 +52,6 @@ confidence: high | image/video/audio/action final type、registry alias、endpoint capability、formatter | [`output-runtime`](rules-output-lifecycle.md#diff-1s-diffusion-output-type-必须从模型声明闭合到-topology-与-formatter)：`DIFF-1s` | `model_metadata.py` → default/multi-stage config → video capability → `output_formatter.py` | | Diffusers/custom/native executor identity、hook、batch capability、frame interpolation | [`output-runtime`](rules-output-lifecycle.md#diff-1aa-diffusers-execution-backend-不得冒充-native-checkpoint-identity)：`DIFF-1aa` | `data.py::uses_diffusers_adapter` → `diffusion_engine.py::supports_request_batch` → registry hook / adapter sampling validation | -| 审查组 | 什么时候触发 | 规则 ID | -|---|---|---| -| `core` | 每次共享 diffusion 审查 | `DIFF-1a`, `DIFF-1b`, `DIFF-1c`, `DIFF-1d`, `DIFF-1e` | -| `execution-parity` | graph/eager、solver、RNG、generator、tensor dtype/device、fused layer、FA determinism、async output/shutdown | `DIFF-1a`–`1j` | -| `checkpoint-distributed` | checkpoint、quantization、HSDP/FSDP、artifact identity、distributed offload、multi-DiT/cache lifecycle | `DIFF-2a`–`2e`, `2p`, `2q`, `2s`, `2y`, `2z` 见 [checkpoint 与加载合同](rules-checkpoint-loading.md)；`2f`–`2j`, `2r`, `2w`, `12a` 见 [component lifecycle](rules-component-lifecycle.md)；`2k` 见 [output/runtime](rules-output-lifecycle.md)；`2l`–`2o`, `2x` 见 [LoRA](rules-lora.md) | -| `quality-evidence` | 质量阈值、offload、A/B case | `DIFF-3a` | -| `distributed-vae` | Wan VAE spatial height/width shard、rank context、extent/padding | `DIFF-3b` | -| `system-runtime` | cache/预算、native/backend/platform、attention layout、SymmMem A2A、能力 metadata、异常与并发 | `DIFF-4a`–`4j`、`4v`、`4x`（`4a`–`4i`、`4x` 见 [paged cache 与系统运行时规则](rules-system-runtime.md)，`4v` 见 [attention rules](rules-attention.md)）；reply ownership `4z` 见 [output/runtime](rules-output-lifecycle.md) | -| [`metrics-evidence`](rules-metrics.md) | diffusion timing、step normalization、missing-versus-zero、benchmark interpretation | `DIFF-5a`, `DIFF-5b` | -| `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `DIFF-0a`, `DIFF-0b` | - ## 优化路径与 eager 的等价合同 ### DIFF-1a — graph/compile/fused 路径逐项复刻 eager 数值边界 diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules.md b/knowledge/repos/vllm-omni/components/model-executor/rules.md index 68244c745..7051e9def 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules.md @@ -11,6 +11,21 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru ## Direct 代码快速入口 +| PR 描述信号 | 规则入口 | +|---|---| +| stage schema、projection、known fields | `EXEC-3a` | +| connector capability、worker/runner selection | `EXEC-3c` | +| runner preprocess、exact-shape input、MTP | `EXEC-4e` 与本页 runner 合同 | +| stage TP/PP/DP、replica、device capacity | `EXEC-3d` | +| runtime info、request RNG、bridge/batch/embedding width | `EXEC-1a`–`EXEC-1i`；[bridge/batch](rules-bridge-batch.md) | +| loader、checkpoint config、snapshot/subdir、packed weights | `EXEC-2a`–`EXEC-2i`；[loader](rules-loader-contract.md) | +| async output、MTP graph、output types | `EXEC-4c`、`EXEC-5a`–`5b`、`EXEC-7a`–`7b` | +| NPU/ROCm/MUSA/XPU runner/backend | `EXEC-10a`–`EXEC-13h`；[platform backends](rules-platform-backends.md) | +| image/video task envelope、`model_extras` | `EXEC-6a`–`EXEC-6d`；[image task envelope](rules-image-task-envelope.md) | +| sampling/codec hot path | `EXEC-8a`、`EXEC-9a`、`EXEC-11a`–`11h`；[runtime hot paths](rules-runtime-hot-paths.md) | + +## 完整代码路由 + - **EXEC-0a — PR 描述先选代码地图。** Direct review 先按 title/body 声明的 runner、stage、bridge、loader 或设备语义命中下表，再用 pinned changed files 验证真实模型 consumer；路径只负责范围反查。 - **EXEC-0b — 共享 producer 先于模型补丁。** 命中共享 runner、stage runtime 或 bridge producer 后立即沿 live consumer 审查；只有 producer 正确且问题只属于一个模型时才进入该模型 owner。 @@ -35,23 +50,6 @@ sources: [vllm_omni/worker/gpu_model_runner.py, vllm_omni/worker/gpu_ar_model_ru | Qwen3-TTS adaptive chunk、EWMA target、all-frame emit、controller cleanup | [Qwen3-TTS adaptive ramp](../../models/qwen3-tts/rules.md#q3tts-3e-adaptive-ramp-是-host-side每段-opt-in-控制器不是-cuda-graph-计划) | `stage_input_processors/qwen3_tts.py::talker2code2wav_async_chunk` → `chunk_size_utils.py::{AdaptiveChunkController,compute_adaptive_emit}` → Code2Wav | | Qwen3-Omni audio encoder 的 head/TP divisibility | [Qwen-Omni audio-encoder TP rule](../../models/qwen-omni/rules.md#qomni-1d-audio-encoder-tp-必须按-head-divisibility-局部回退) | `qwen3_omni_moe_thinker.py::{Qwen3OmniMoeAudioAttention,Qwen3OmniMoeAudioEncoderLayer}`；只影响该 encoder layer 的 QKV/output/FFN TP fallback ^[PR #4322] | -| 审查组 | 什么时候触发 | 规则 ID | -|---|---|---| -| `core` | 每次 model-executor 审查 | `EXEC-1a` | -| `strict-stage-config` | stage schema、projection、known fields | `EXEC-3a` | -| `connector-capability` | full-payload/connector stage 的 worker runner validation | `EXEC-3c` | -| `stage-runtime` | stage 数、replica、inline/subprocess 与设备容量 | `EXEC-3d` + 本页 stage 并行度验收 | -| `bridge-batch` | runtime info、跨 stage payload、batch、request RNG | `EXEC-1a`–`EXEC-1i`，见 [跨 stage bridge 与 batch 合同](rules-bridge-batch.md) | -| `loader-contract` | dtype、checkpoint config 获取、loader、fused shard 拼装、stage snapshot/subdir、wrapper callable 委托 | `EXEC-2a`–`EXEC-2i`，见 [loader 合同](rules-loader-contract.md) | -| `async-output` | AR async output、snapshot/live state、平台与 guard/fallback | `EXEC-5a`, `EXEC-5b` | -| `mtp-graph` | Talker-MTP FULL graph、平台能力与 tri-state fallback | `EXEC-4c` | -| `runner-preprocess` | graph-bucket padded `input_ids` 与 semantic token-count metadata | `EXEC-4e` | -| `image-task-envelope` | shared image/video task example、video output consumer 或 `model_extras` prompt builder | `EXEC-6a`–`EXEC-6d`，见 [image task envelope 合同](rules-image-task-envelope.md) | -| `author-routing` | 只供 Direct reviewer 导航，不作为 finding 规则 | `EXEC-0a`, `EXEC-0b` | -| `output-contract` | Omni 输出类型与字段/复制合同 | `EXEC-7a`–`EXEC-7b`，见 [输出类型合同](rules-output-contract.md) | -| `runtime-hot-paths` | 采样循环不变量、固定输入缓存、AR 音频侧路、codec 帧账本与 CosyVoice3 typed handoff | `EXEC-8a`, `EXEC-9a`, `EXEC-11a`–`EXEC-11h`，见 [运行时热路径合同](rules-runtime-hot-paths.md) | -| `platform-backends` | NPU runner 接口、ROCm 分页注意力、NPU 模型补丁注册 | `EXEC-10a`–`EXEC-10e`, `EXEC-12a`–`EXEC-13h`，见 [平台后端合同](rules-platform-backends.md) | - ## 严格配置校验 ### EXEC-3a — 严格配置校验不能靠扩大白名单 diff --git a/knowledge/repos/vllm-omni/models/bagel/rules.md b/knowledge/repos/vllm-omni/models/bagel/rules.md index dfd2ca056..478792595 100644 --- a/knowledge/repos/vllm-omni/models/bagel/rules.md +++ b/knowledge/repos/vllm-omni/models/bagel/rules.md @@ -10,6 +10,12 @@ confidence: high # BAGEL 实现规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| CFG position ID、nested DiT owner、step image wave/request state | `BAGEL-1`–`BAGEL-3` | + ## BAGEL-1：CFG position ID 沿序列轴合并 **适用范围**：`Bagel.forward` 将顺序 CFG 的 gen/cfg_text/cfg_img 分支合并为一次 diff --git a/knowledge/repos/vllm-omni/models/boogu-image/rules.md b/knowledge/repos/vllm-omni/models/boogu-image/rules.md index b1e2382ea..1046549fc 100644 --- a/knowledge/repos/vllm-omni/models/boogu-image/rules.md +++ b/knowledge/repos/vllm-omni/models/boogu-image/rules.md @@ -10,6 +10,12 @@ confidence: high # Boogu-Image 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| revision、RoPE、Edit CFG/rank/VAE、request batching | `BOOGU-1a`–`BOOGU-1d` | + ## BOOGU-1a — global revision 必须填入每个未显式设 revision 的 stage - 触发：全局/stage checkpoint revision handoff、Boogu component loader 或 Edit-Turbo recipe 变更。 diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md b/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md index 3aa8086c1..985107088 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md @@ -10,6 +10,12 @@ confidence: high # CosyVoice3 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| TensorRT CFM stream handoff 或 flow-estimator plan publication | `COSYVOICE3-1a`、`COSYVOICE3-1b` | + ## COSYVOICE3-1a — TensorRT CFM 的跨 stream handoff 必须同时守住顺序和 allocator lifetime - 触发：修改 CosyVoice3 `ConditionalCFM.forward_estimator` 的非-`torch.nn.Module` diff --git a/knowledge/repos/vllm-omni/models/dots-tts/rules.md b/knowledge/repos/vllm-omni/models/dots-tts/rules.md index ac50a345b..8b22208fb 100644 --- a/knowledge/repos/vllm-omni/models/dots-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/dots-tts/rules.md @@ -10,6 +10,12 @@ confidence: high # dots.tts 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| weekly E2E、prompt scaffold、result queue 或 request-noise oracle | `DOTS-1a` | + ## DOTS-1a — weekly E2E scaffold 必须保留 prompt、队列与 request-noise oracle - 触发：修改 dots.tts offline E2E、prompt helper、`compute_logits` result queue 或 flow-matching noise test。 diff --git a/knowledge/repos/vllm-omni/models/flux2-klein/rules.md b/knowledge/repos/vllm-omni/models/flux2-klein/rules.md index 1b749d04e..8b4520f9f 100644 --- a/knowledge/repos/vllm-omni/models/flux2-klein/rules.md +++ b/knowledge/repos/vllm-omni/models/flux2-klein/rules.md @@ -10,6 +10,12 @@ confidence: high # FLUX.2-Klein 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| final-layout HWR、packed weights 或 warm restore validation | `FLUXK-1a` | + 只有 `FLUXK-数字字母` 是可审计规则 ID。共享 artifact、lease 和 DLO transport 约束见 [Diffusion checkpoint rules](../../components/diffusion/rules-checkpoint-loading.md#diff-2z-final-layout-hwr-只能接入-eligible-no-allgather-dlo) 与 [Host Weight Runtime rules](../../components/host-weight-runtime/rules.md)。 diff --git a/knowledge/repos/vllm-omni/models/hidream-o1-image/rules.md b/knowledge/repos/vllm-omni/models/hidream-o1-image/rules.md index 0285be6d7..aad402914 100644 --- a/knowledge/repos/vllm-omni/models/hidream-o1-image/rules.md +++ b/knowledge/repos/vllm-omni/models/hidream-o1-image/rules.md @@ -10,6 +10,12 @@ confidence: high # HiDream-O1 Image 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| request geometry、mixed attention、TP、Cache-DiT 或 patch output | `HIDREAMO1-1a` | + HiDream-O1 Image 的模型专属约束集中在本页；模型 registry signature、共享 attention backend 及通用 Cache-DiT lifecycle 不在此重复展开。 ## HIDREAMO1-1a — HiDream-O1 Image pipeline 请求、并行与输出合同 diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/rules.md b/knowledge/repos/vllm-omni/models/higgs-audio/rules.md index 211e087b5..38ba1001d 100644 --- a/knowledge/repos/vllm-omni/models/higgs-audio/rules.md +++ b/knowledge/repos/vllm-omni/models/higgs-audio/rules.md @@ -10,6 +10,13 @@ confidence: high # Higgs-Audio 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| request RNG/decode state | `HIGGS-1a` | +| voice-clone sentinel、placeholder positions 或 chunked prefill | `HIGGS-2a` | + ## HIGGS-1a — V3 codec sampling 与 decode state 必须按 request 所有 - 触发：Higgs-Audio V3 seeded sampling、八 codebook flatten/compaction、GPU decode delay 或 replacement request。 diff --git a/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md b/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md index 7228585fb..2f243852a 100644 --- a/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md +++ b/knowledge/repos/vllm-omni/models/indextts-2-5/rules.md @@ -10,6 +10,12 @@ confidence: high # IndexTTS 2.5 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| bundle/topology、frontend/voice、证据边界或 S2Mel batching | `ITTS25-1a`–`ITTS25-1d` | + ## ITTS25-1a — native bundle/config 与两 stage topology 必须 fail fast - 触发：修改 registry、pipeline、deploy、bundle discovery、tokenizer-free init 或 latent policy。 diff --git a/knowledge/repos/vllm-omni/models/longcat-image/rules.md b/knowledge/repos/vllm-omni/models/longcat-image/rules.md index aa71c5588..b728c5a0b 100644 --- a/knowledge/repos/vllm-omni/models/longcat-image/rules.md +++ b/knowledge/repos/vllm-omni/models/longcat-image/rules.md @@ -10,6 +10,12 @@ confidence: high # LongCat-Image 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| Edit request geometry、conditioning resize、latent/decode propagation | `LONGCAT-1a` | + 只有 `LONGCAT-数字字母` 是可审计规则 ID。 ## LONGCAT-1a — Edit 请求几何必须单一解析并端到端传播 diff --git a/knowledge/repos/vllm-omni/models/ltx2/rules.md b/knowledge/repos/vllm-omni/models/ltx2/rules.md index d683c2378..6f2f149b4 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/rules.md +++ b/knowledge/repos/vllm-omni/models/ltx2/rules.md @@ -10,7 +10,14 @@ confidence: high # LTX-2.5 DiffVAE 规则 -### LTX25-1 — LTX-2.5 的 decoder、统计量与 artifact 必须作为同一版本化合同维护 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| DiffVAE artifact/version 或 request/distributed decode | `LTX25-1`、`LTX25-2` | +| decoded-video transport、device quantization 或 profiler boundary | `LTX-3` | + +## LTX25-1 — LTX-2.5 的 decoder、统计量与 artifact 必须作为同一版本化合同维护 - 触发：修改 LTX-2.5 full/distilled one- or two-stage pipeline 的视频 decoder、`ltx2_use_conv_vae`、 latent normalization、Native sidecar、component residency 或 NATTEN Hub-kernel dependency/remediation @@ -31,7 +38,7 @@ confidence: high guidance、production error 和 LTX-2.5 test 必须一致地说明 `kernels==0.15.2` 与上述 Transformers 范围。^[PR #6189] ^[PR #7020] -### LTX25-2 — DiffVAE 请求边界与分布式 decode 必须显式 fail-closed +## LTX25-2 — DiffVAE 请求边界与分布式 decode 必须显式 fail-closed - 触发：修改 LTX-2.5 decode、`vae_parallel_mode`、tiling、request resolution validation 或 request RNG。 - 强制：DiffVAE request validation 从首个 decoder-stage kernel 推导最小 latent H/W，并在 denoising 前 @@ -45,7 +52,7 @@ confidence: high - 验收：覆盖 early geometry rejection 与 ConvVAE control、tile positive/non-tile negative modes、serial RNG-order parity，以及 output/non-output rank 在 collective/non-collective decode 的分支。^[PR #6189] -### LTX-3 — LTX decoded-video transport 必须在设备端量化且保持 presentation 与 profiler 边界 +## LTX-3 — LTX decoded-video transport 必须在设备端量化且保持 presentation 与 profiler 边界 - 触发：修改 LTX-2/2.3/2.5 decoded-video finalization、`output_type="np"`、worker/IPC transport，或 LTX diffusion-pipeline profiler wrapping。 diff --git a/knowledge/repos/vllm-omni/models/mage-vl/rules.md b/knowledge/repos/vllm-omni/models/mage-vl/rules.md index 75858498c..fbe810aec 100644 --- a/knowledge/repos/vllm-omni/models/mage-vl/rules.md +++ b/knowledge/repos/vllm-omni/models/mage-vl/rules.md @@ -10,6 +10,12 @@ confidence: high # Mage-VL 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| experimental adapter、bounded duplex、codec backend 或 native-support claim | `MAGEVL-1a` | + ## MAGEVL-1a — experimental adapter 必须保持 remote-code、bounded duplex 与 non-native boundary - 触发：Mage-VL loader/backend、adapter capability、visual window/gate flow、WebSocket protocol、codec backend、 diff --git a/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md b/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md index 7ca6d1c58..d4b5cca86 100644 --- a/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md +++ b/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md @@ -10,6 +10,13 @@ confidence: high # MammothModa2 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| task-scoped AR vocabulary | `MAMMO-1a` | +| AR→DiT token/hidden-state bridge | `MAMMO-1b` | + ## MAMMO-1a — AR sampling 必须按 task 隔离 text 与 visual vocabulary - 触发：修改 Preview/Qwen2.5-VL 或 Dev/Qwen3-VL 的 T2I prompt、额外 generation vocabulary、 diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 2da88ff7e..5dc86aa3e 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -14,6 +14,17 @@ confidence: high ## Direct 代码快速入口 +| PR 描述信号 | 规则入口 | +|---|---| +| version/registry、processor、remote code | `MCPMO-1a`、`MCPMO-2a` | +| Whisper/APM 或 SigLIP attention | `MCPMO-2b`、`MCPMO-2c` | +| TTS/Code2Wav/TRT、CUDA Graph | `MCPMO-1b`–`MCPMO-1e`；[CUDA graphs](rules-cuda-graphs.md) | +| batch、runtime info、reference-audio handoff | `MCPMO-3a`–`MCPMO-3e`；[Code2Wav batching](rules-code2wav-batching.md) | +| native duplex、sampling budget、shipping profile、playback ACK | `MCPMO-4a`–`MCPMO-4h`；[duplex](rules-duplex.md) | +| accuracy/performance/online-serving evidence | `MCPMO-5a`、`MCPMO-5b` | + +## 完整代码路由 + | PR 描述信号 | 规则组 | 第一批源码 | |---|---|---| | MiniCPM-o 4.5、`minicpmo_4_5`、版本识别 | MCPMO-2a | `config/pipeline_registry.py::OMNI_PIPELINES["minicpmo_4_5"]` → `model_executor/models/minicpmo_4_5/pipeline.py`；`model_executor/models/registry.py::_OMNI_MODELS` | @@ -32,9 +43,6 @@ confidence: high | duplex Talker chunk、runtime `min_tokens`、chat sampling isolation | [`MCPMO-4c`](rules-duplex.md#mcpmo-4c-native-duplex-talker-必须按-generate_chunk-预算终止) | `experimental/fullduplex/minicpmo45/adapter.py` → stage 1 sampling → Talker `generate_chunk` | | shipping YAML、duplex/chat 共服、session capacity、async scheduler | [`MCPMO-4e`](rules-duplex.md#mcpmo-4e-shipping-profile-必须共服-chat-与-native-duplex) | `deploy/minicpmo_4_5*.yaml` → config merge → realtime/chat entrypoints | | playback ACK、response-owned assistant history、下一 input 后的 late ACK | [`MCPMO-4h`](rules-duplex.md#mcpmo-4h-responseplayback-history-必须由回答它的-input-commit-所有) | `experimental/fullduplex/openai/{protocol,serving}.py` → response snapshot / conversation item lifecycle | -| Daily-Omni/Seed-TTS accuracy、simplex/duplex perf、TTFT/TTFP/RTF | MCPMO-5a | `tests/e2e/accuracy/minicpmo_4_5/`、`tests/dfx/perf/tests/test_minicpmo_4_5*.json` → `run_benchmark.py` | -| online serving CI、async/sync chunk matrix、duplex fixture、soft-interrupt event oracle、local ref audio | MCPMO-5b | `tests/e2e/online_serving/{run_minicpmo_realtime_duplex_soft_interrupt.py,test_minicpmo_realtime_duplex_drivers.py}` → `helpers/minicpmo_4_5_duplex.py` | - ## MCPMO-1a — trust_remote_code 服从用户选择 - 触发：API/CLI server 为 MiniCPM-o 加载 tokenizer、processor、config 或模型代码。 diff --git a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md index ce8a7720a..08f14d73a 100644 --- a/knowledge/repos/vllm-omni/models/minimax-h3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-h3/rules.md @@ -14,6 +14,17 @@ confidence: high ## Direct 代码快速入口 +| PR 描述信号 | 规则入口 | +|---|---| +| component quantization、weight loading、HWR | `MMH3-1a`、`MMH3-1f`；[loading](rules-loading.md) | +| attention/RoPE/backend、encoder process state | `MMH3-1c`–`MMH3-1e`、`MMH3-1g`、`MMH3-1k`、`MMH3-1n` | +| media ingress、reference tasks、split text encoder | `MMH3-2a`、`MMH3-2b`、`MMH3-2l`；[media](rules-media.md) | +| modular task、cache policy、schedules、LoRA/FastH3 | `MMH3-2c`–`MMH3-2n`；[cache/task](rules-cache-task.md) | +| DLO、hardware recipes、CI/perf evidence | `MMH3-3a`–`MMH3-3m`；[deployment](rules-deployment.md) | +| VAE kernels、materialization、fallback | `MMH3-4c`；[VAE ops](rules-vae-ops.md) | + +## 完整代码路由 + | PR 描述信号 | 规则 | 第一批源码 | |---|---|---| | online FP8、`ignored_layers`、component prefix | [MMH3-1a](rules-loading.md#mmh3-1a-component-namespace-与-checkpoint-transform-必须在-active-loader-前闭合) | `pipeline_minimax_h3.py::_resolve_component_quant_config` → `MiniMaxH3DiTModel` linear prefix | diff --git a/knowledge/repos/vllm-omni/models/minimax-music3/rules.md b/knowledge/repos/vllm-omni/models/minimax-music3/rules.md index 1d5c7a1e3..83b216ef1 100644 --- a/knowledge/repos/vllm-omni/models/minimax-music3/rules.md +++ b/knowledge/repos/vllm-omni/models/minimax-music3/rules.md @@ -10,6 +10,12 @@ confidence: high # MiniMax Music3 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| composite checkpoint root、component completeness 或 snapshot revision | `MM3-1a` | + ## MM3-1a — composite checkpoint 的直接组件必须与 AR snapshot 同 revision 且完整 - 触发：MiniMax-Music3 从 Hub/cache/root/`language_model/` 初始化 AR 或 acoustic stage、恢复 repo diff --git a/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md b/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md index 155a4bde2..e8590f6ec 100644 --- a/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md +++ b/knowledge/repos/vllm-omni/models/nemotron-voicechat/rules.md @@ -10,6 +10,12 @@ confidence: high # Nemotron VoiceChat 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| duplex data plane、tool result、codec lifecycle 或 deployment evidence | `NVC-1a`–`NVC-1f` | + 只有 `NVC-数字字母` 是本页可审计规则 ID。 ## NVC-1a — 原生 duplex 是 frame-locked、可恢复的三 stage 数据面 diff --git a/knowledge/repos/vllm-omni/models/personaplex/rules.md b/knowledge/repos/vllm-omni/models/personaplex/rules.md index 7e850e56a..bce247b2c 100644 --- a/knowledge/repos/vllm-omni/models/personaplex/rules.md +++ b/knowledge/repos/vllm-omni/models/personaplex/rules.md @@ -10,6 +10,14 @@ confidence: high # PersonaPlex 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| frame/de-delay、session codec、persona/voice mutation | `PPLEX-1a`–`PPLEX-1c` | +| standalone lease | `PPLEX-2a` | +| checkpoint config 或 sampling capability | `PPLEX-3a`、`PPLEX-3b` | + 只有 `PPLEX-数字字母` 是可审计规则 ID。 ## PPLEX-1a — frame 与 de-delay 边界必须跨 chunk 连续 diff --git a/knowledge/repos/vllm-omni/models/pi0/rules.md b/knowledge/repos/vllm-omni/models/pi0/rules.md index 0391c9188..3d65e8b94 100644 --- a/knowledge/repos/vllm-omni/models/pi0/rules.md +++ b/knowledge/repos/vllm-omni/models/pi0/rules.md @@ -10,6 +10,12 @@ confidence: high # Pi0 模型硬门禁 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| observation preprocessing、flow matching、attention 或 action output | `PI0-1a` | + 本页承载 Pi0 专有的 VLA 观测预处理、flow-matching kernel、attention 边界与动作输出验证合同；共享输出和配置 wiring 规则见相关页面。 ## PI0-1a — π0 的观测、attention 与动作输出必须保持 OpenPI/LeRobot 合同 diff --git a/knowledge/repos/vllm-omni/models/qwen-image/rules.md b/knowledge/repos/vllm-omni/models/qwen-image/rules.md index 55b31c4aa..127e44867 100644 --- a/knowledge/repos/vllm-omni/models/qwen-image/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-image/rules.md @@ -10,6 +10,13 @@ confidence: high # Qwen-Image 实现规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| accuracy/repeatability gate | `QWENIMG-1a` | +| compiled complex RoPE | `QWENIMG-1b` | + ## QWENIMG-1a — accuracy gate 必须先证明同 seed repeatability - 触发：修改 Qwen-Image accuracy threshold、regional compile 或 fixture 的 deterministic opt-in。 @@ -27,4 +34,3 @@ confidence: high - 强制：对 `vid_freqs` 和 `txt_freqs` 使用 `torch.real()`/`torch.imag()` 后再转换为 query 对应 dtype；即使问题由 MUSA 暴露，也保持 CUDA 与其他平台使用同一等价函数式表达式。 - 禁止：在该编译路径恢复 `freqs.real`/`freqs.imag` 属性链，或仅用 `--enforce-eager` 掩盖重复 tensor alias guard；不得把该模型级修复泛化为所有 diffusion RoPE caller 已具备同等保障。 - 验收：以 legacy 属性表达式作为 reference，验证函数式路径的值与 dtype 精确一致；在目标 MUSA 上以 Qwen Image 默认编译配置验证不再出现 `Duplicate tensors found`，并在 CUDA smoke 中确认输出 parity。^[PR #6110] - diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index ec204096d..83d81c91c 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -10,6 +10,12 @@ confidence: high # Qwen-Omni 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| MRoPE、quantization mapping、pipeline key、audio TP、code2wav 或 backend defaults | `QOMNI-1a`–`QOMNI-1f` | + ## QOMNI-1a — Qwen3-Omni Thinker MRoPE 必须保留 CUDA custom-op 边界 - 触发：修改 `qwen3_omni_moe.yaml` 的 CUDA compilation 配置、Thinker MRoPE/RMSNorm 路径，或升级会改变 Inductor MRoPE fusion 的 vLLM 版本。 diff --git a/knowledge/repos/vllm-omni/models/sana-video/rules.md b/knowledge/repos/vllm-omni/models/sana-video/rules.md index 9380e30c3..7a31a5889 100644 --- a/knowledge/repos/vllm-omni/models/sana-video/rules.md +++ b/knowledge/repos/vllm-omni/models/sana-video/rules.md @@ -10,6 +10,13 @@ confidence: high # SANA-Video 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| TP/CFG topology | `SANA-1a` | +| Cache-DiT/offload 或 request refresh steps | `SANA-1b`、`SANA-1c` | + ## SANA-1a — native SANA Video 只支持 TP/CFG 的 {1,2} 矩阵 - 触发：native SANA Video TP/CFG、transformer layout、I2V 或 Diffusers adapter topology 变更。 diff --git a/knowledge/repos/vllm-omni/models/step-audio2/rules.md b/knowledge/repos/vllm-omni/models/step-audio2/rules.md index 2cc9672b3..31e674519 100644 --- a/knowledge/repos/vllm-omni/models/step-audio2/rules.md +++ b/knowledge/repos/vllm-omni/models/step-audio2/rules.md @@ -9,6 +9,13 @@ sources: ["PR #5067", vllm_omni/model_executor/models/step_audio2/step_audio2_th # Step-Audio2 设备边界与流式性能规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| streaming token ownership 或 ragged audio features | `STEPA2-1a`、`STEPA2-1b` | +| shared audio runtime 或 DiT TensorRT plan publication | `STEPA2-2a`、`STEPA2-2b` | + 以下规则在 `main @ 12a5f6fb` 复核；主要约束 Step-Audio2 的 thinker/token2wav 路径。共享 `Token2Wav` core 的 blast radius 在下文显式列出，runtime 的通用规则 仍归组件 owner。 diff --git a/knowledge/repos/vllm-omni/models/voxtral-tts/rules.md b/knowledge/repos/vllm-omni/models/voxtral-tts/rules.md index 27e8c21ac..d81d0d3bc 100644 --- a/knowledge/repos/vllm-omni/models/voxtral-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/voxtral-tts/rules.md @@ -10,6 +10,12 @@ confidence: high # Voxtral-TTS hot-path 规则 +## Direct 代码快速入口 + +| PR 描述信号 | 规则 | +|---|---| +| graph layering、host sync 或 per-request CFG metadata | `VOXTTS-1a`–`VOXTTS-1c` | + 只有 `VOXTTS-数字字母` 是可审计规则 ID。 ## VOXTTS-1a — variable-output full graph 与 inner acoustic graph 分层 diff --git a/test/test_v2_p0.py b/test/test_v2_p0.py index 0a4fdd967..1b120fcdf 100644 --- a/test/test_v2_p0.py +++ b/test/test_v2_p0.py @@ -301,7 +301,8 @@ def test_adapter_zero_declares_risk_tiers(): # process-wide defaults, diffusion is multi-model shared code, and # distributed owns cross-stage transport — small diffs there still # need the full review ensemble - "platform", "diffusion", "distributed"} + "platform", "diffusion", "distributed", + "runtime_patch", "host_weight_runtime"} # -- repo-neutral core guard (§V2.2.1) -------------------------------------------