From 7747ee2fe0dc58b9d6ce2c051fe6a40efe3c84e0 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Wed, 10 Jun 2026 18:34:37 +1000 Subject: [PATCH 1/3] feat(ai-openrouter): video generation adapter (/api/v1/videos) + image activity follow-ups Closes #707. - Add openRouterVideo: async jobs adapter for OpenRouter's dedicated video API (submit -> poll -> download). Per-model size/duration/option types are generated from GET /api/v1/videos/models; frame roles map onto frame_images[] / input_references[] per the MediaInputRole taxonomy. - Teach the model-meta sync scripts the videos/models endpoint (openrouter.video-models.json + OPENROUTER_VIDEO_MODEL_META). - Image adapter follow-ups from the #624 review: throw on unmapped sizes (the size union used a Unicode multiplication sign so every non-square size silently dropped its aspect ratio), throw on numberOfImages > 1 (live-verified: the gateway ignores all count keys), expose image_config.strength. - Completed videos are returned as data: URLs (unsigned_urls 401 without the API key header) with gateway-reported cost on usage.cost. The SDK's getVideoContent is bypassed: its matcher only accepts application/octet-stream while the endpoint serves video/mp4. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/openrouter-video-adapter.md | 7 + docs/adapters/openrouter.md | 85 ++ docs/config.json | 7 +- docs/media/image-generation.md | 2 +- docs/media/video-generation.md | 69 +- packages/ai-openrouter/src/adapters/image.ts | 42 +- packages/ai-openrouter/src/adapters/video.ts | 423 ++++++++++ .../src/image/image-provider-options.ts | 26 +- packages/ai-openrouter/src/index.ts | 15 + packages/ai-openrouter/src/model-meta.ts | 255 +++++- .../src/video/video-provider-options.ts | 157 ++++ .../ai-openrouter/tests/image-adapter.test.ts | 97 ++- .../ai-openrouter/tests/video-adapter.test.ts | 488 ++++++++++++ .../skills/ai-core/media-generation/SKILL.md | 42 +- scripts/convert-openrouter-models.ts | 51 +- scripts/fetch-openrouter-models.ts | 77 +- scripts/openrouter.video-models.json | 745 ++++++++++++++++++ scripts/openrouter.video-models.ts | 45 ++ testing/e2e/src/lib/feature-support.ts | 9 +- 19 files changed, 2574 insertions(+), 68 deletions(-) create mode 100644 .changeset/openrouter-video-adapter.md create mode 100644 packages/ai-openrouter/src/adapters/video.ts create mode 100644 packages/ai-openrouter/src/video/video-provider-options.ts create mode 100644 packages/ai-openrouter/tests/video-adapter.test.ts create mode 100644 scripts/openrouter.video-models.json create mode 100644 scripts/openrouter.video-models.ts diff --git a/.changeset/openrouter-video-adapter.md b/.changeset/openrouter-video-adapter.md new file mode 100644 index 000000000..d1b9e7f7d --- /dev/null +++ b/.changeset/openrouter-video-adapter.md @@ -0,0 +1,7 @@ +--- +'@tanstack/ai-openrouter': minor +--- + +Add `openRouterVideo`, a video generation adapter for OpenRouter's dedicated async API (`POST /api/v1/videos`) — Seedance, Veo 3.1, Wan, Kling, and Sora 2 Pro through one API key. Follows the jobs/polling architecture (`generateVideo()` → `getVideoJobStatus()`), with per-model `size` / `duration` / provider-option types generated from OpenRouter's `GET /api/v1/videos/models` metadata and validated before submit. Image-conditioned prompts map `metadata.role` onto the wire: `start_frame` / `end_frame` → `frame_images[]` (`first_frame` / `last_frame`), `reference` / `character` → `input_references[]`; frame roles are validated against each model's `supported_frame_images`. Completed videos are downloaded server-side and returned as `data:` URLs (OpenRouter's download URLs require the API key), and the gateway-reported cost is surfaced as `usage.cost`. + +Image adapter fixes from the #624 review: requested `size` is now validated (the `WIDTHxHEIGHT` union previously used a Unicode `×`, so every size except `1024x1024` silently dropped its aspect ratio; unsupported sizes now throw with the supported list), `numberOfImages > 1` throws instead of silently returning one image (verified live: the gateway ignores all count keys in `image_config`), and `image_config.strength` (0.0–1.0 image-to-image influence) is exposed via `modelOptions.strength`. diff --git a/docs/adapters/openrouter.md b/docs/adapters/openrouter.md index cd8984511..eff6d0aec 100644 --- a/docs/adapters/openrouter.md +++ b/docs/adapters/openrouter.md @@ -219,6 +219,91 @@ fields are simply absent and the stream completes normally. Both `openRouterText` and `openRouterResponsesText` populate cost when OpenRouter returns it. +## Image Generation + +`openRouterImage` routes image generation through OpenRouter's +chat-completions surface (`modalities: ['image']`). Multimodal prompts are +supported — text and image parts are forwarded in order for +image-conditioned generation: + +```typescript +import { generateImage } from "@tanstack/ai"; +import { openRouterImage } from "@tanstack/ai-openrouter"; + +const result = await generateImage({ + adapter: openRouterImage("google/gemini-2.5-flash-image"), + prompt: "A watercolor lighthouse at dusk", + size: "1344x768", // mapped to image_config.aspect_ratio ('16:9') + modelOptions: { + image_size: "2K", // resolution (Gemini models) + strength: 0.35, // image-to-image influence, i2i-capable models only + }, +}); +``` + +Notes: + +- The pathway returns **exactly one image per request** — `numberOfImages > 1` + throws instead of silently under-delivering. Make multiple requests if you + need multiple candidates. +- `size` must be one of the ten supported `WIDTHxHEIGHT` values (it is + converted to `image_config.aspect_ratio`); anything else throws with the + supported list. + +## Video Generation (Experimental) + +`openRouterVideo` targets OpenRouter's dedicated **async video API** +(`POST /api/v1/videos`) — Seedance, Veo 3.1, Wan, Kling, and Sora 2 Pro +through your one OpenRouter key. It follows the jobs/polling architecture +shared by all TanStack AI video adapters: + +```typescript +// Server: create the job, then poll +import { generateVideo, getVideoJobStatus } from "@tanstack/ai"; +import { openRouterVideo } from "@tanstack/ai-openrouter"; + +const adapter = openRouterVideo("bytedance/seedance-2.0"); + +const { jobId } = await generateVideo({ + adapter, + prompt: [ + { type: "text", content: "Animate this product shot, slow push-in" }, + { + type: "image", + source: { type: "url", value: "https://your-cdn.com/product.png" }, + metadata: { role: "start_frame" }, + }, + ], + size: "1280x720", + duration: 8, +}); + +let status = await getVideoJobStatus({ adapter, jobId }); +while (status.status !== "completed" && status.status !== "failed") { + await new Promise((r) => setTimeout(r, 5000)); + status = await getVideoJobStatus({ adapter, jobId }); +} +// status.url is a data: URL (OpenRouter download URLs require the API key, +// so the adapter downloads server-side); status.usage?.cost is the real +// billed cost reported by the gateway. +``` + +```tsx +// Client: track the job with the useGenerateVideo hook +import { useGenerateVideo, fetchServerSentEvents } from "@tanstack/ai-react"; + +const { generate, result, videoStatus, isLoading } = useGenerateVideo({ + connection: fetchServerSentEvents("/api/generate/video"), +}); +// result?.url renders directly: