diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index ff91e738ed..57026dee82 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -2683,4 +2683,128 @@ describe("initSandboxRuntimeModular", () => { }).not.toThrow(); }); }); + + describe("audio buffer pre-decode at mount", () => { + it("warm-decodes every timed audio source before any play()", async () => { + const decodeAudioData = vi.fn(async () => ({}) as AudioBuffer); + class MockAudioContext { + currentTime = 0; + state = "running"; + destination = {}; + decodeAudioData = decodeAudioData; + createGain() { + return { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() }; + } + resume = vi.fn(); + close = vi.fn(); + } + vi.stubGlobal("AudioContext", MockAudioContext); + const fetchMock = vi.fn(async () => ({ + ok: true, + arrayBuffer: async () => new ArrayBuffer(8), + })); + vi.stubGlobal("fetch", fetchMock); + + try { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-duration", "10"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + const audioA = document.createElement("audio"); + audioA.setAttribute("data-start", "0"); + audioA.setAttribute("data-duration", "5"); + audioA.setAttribute("src", "https://media.example/a.mp3"); + const audioB = document.createElement("audio"); + audioB.setAttribute("data-start", "5"); + audioB.setAttribute("data-duration", "5"); + audioB.setAttribute("src", "https://media.example/b.mp3"); + root.append(audioA, audioB); + document.body.appendChild(root); + window.__timelines = { main: createMockTimeline(10) }; + + initSandboxRuntimeModular(); + + // No play() has happened — the decode must be driven by mount alone. + await vi.waitFor(() => { + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(decodeAudioData).toHaveBeenCalledTimes(2); + }); + const urls = fetchMock.mock.calls.map((call) => String(call[0])); + expect(urls.some((u) => u.endsWith("a.mp3"))).toBe(true); + expect(urls.some((u) => u.endsWith("b.mp3"))).toBe(true); + } finally { + vi.unstubAllGlobals(); + } + }); + + it("only warms clips near the playhead, and honors the embedder opt-out", async () => { + const decodeAudioData = vi.fn(async () => ({}) as AudioBuffer); + class MockAudioContext { + currentTime = 0; + state = "running"; + destination = {}; + decodeAudioData = decodeAudioData; + createGain() { + return { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() }; + } + resume = vi.fn(); + close = vi.fn(); + } + vi.stubGlobal("AudioContext", MockAudioContext); + const fetchMock = vi.fn(async () => ({ + ok: true, + arrayBuffer: async () => new ArrayBuffer(8), + })); + vi.stubGlobal("fetch", fetchMock); + + try { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-duration", "600"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + const near = document.createElement("audio"); + near.setAttribute("data-start", "0"); + near.setAttribute("data-duration", "5"); + near.setAttribute("src", "https://media.example/near.mp3"); + const far = document.createElement("audio"); + far.setAttribute("data-start", "300"); + far.setAttribute("data-duration", "5"); + far.setAttribute("src", "https://media.example/far.mp3"); + root.append(near, far); + document.body.appendChild(root); + window.__timelines = { main: createMockTimeline(600) }; + + initSandboxRuntimeModular(); + + // Warming a whole long composition costs a full-file fetch + ~23MB of + // PCM per stereo minute PER SOURCE — only playhead-proximate clips may + // pre-decode; the rest stay lazy until the playhead approaches. + await vi.waitFor(() => { + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + expect(String(fetchMock.mock.calls[0]?.[0])).toContain("near.mp3"); + + window.__hfRuntimeTeardown?.(); + fetchMock.mockClear(); + ( + window as Window & { __HF_AUDIO_PREDECODE_DISABLED?: boolean } + ).__HF_AUDIO_PREDECODE_DISABLED = true; + initSandboxRuntimeModular(); + // A document that exists but is never played (e.g. a double-buffered + // preview's standby iframe) must not warm anything. + await new Promise((r) => setTimeout(r, 20)); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + delete (window as Window & { __HF_AUDIO_PREDECODE_DISABLED?: boolean }) + .__HF_AUDIO_PREDECODE_DISABLED; + vi.unstubAllGlobals(); + } + }); + }); }); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index cf65ecd262..b48f709aa4 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -174,6 +174,9 @@ export function initSandboxRuntimeModular(): void { let webAudioReady = false; void webAudio.init().then((ok) => { webAudioReady = ok; + // Warm the decoded-buffer cache as soon as the transport exists (decode + // works on a suspended AudioContext, so no user gesture is needed). + if (ok) predecodeAudioClipBuffers(); }); // `_auto` is a Studio-internal keyframe marker (an auto-tracked endpoint the // parser reads back), NOT an animatable property. Register it as a no-op GSAP @@ -1808,12 +1811,49 @@ export function initSandboxRuntimeModular(): void { } }; + // Boundary-readiness telemetry: surface playback stalls (`waiting`/`stalled` + // on a timed media element while the transport plays) so cut-boundary tuning + // can target the right layer. Bounded per document to avoid diagnostic spam. + let mediaStallDiagnosticsPosted = 0; + const MAX_MEDIA_STALL_DIAGNOSTICS = 20; + const onMediaStallEvent = (event: Event) => { + const el = event.currentTarget; + if (!(el instanceof HTMLMediaElement)) return; + if (!state.isPlaying || state.tornDown) return; + if (mediaStallDiagnosticsPosted >= MAX_MEDIA_STALL_DIAGNOSTICS) return; + mediaStallDiagnosticsPosted += 1; + let bufferedEnd: number | null = null; + try { + bufferedEnd = el.buffered.length > 0 ? el.buffered.end(el.buffered.length - 1) : null; + } catch { + // buffered ranges unavailable + } + postRuntimeMessage({ + source: "hf-preview", + type: "diagnostic", + code: "runtime_media_stall", + details: { + event: event.type, + tagName: el.tagName.toLowerCase(), + currentSrc: el.currentSrc || null, + readyState: el.readyState, + networkState: el.networkState, + mediaTime: el.currentTime, + bufferedEnd, + compositionTime: state.currentTime, + clipStart: Number.parseFloat(el.dataset.start ?? "") || null, + }, + }); + }; + const unbindMediaMetadataListeners = () => { for (const mediaEl of metadataBoundMedia) { mediaEl.removeEventListener("loadedmetadata", scheduleMetadataDurationHydration); mediaEl.removeEventListener("durationchange", scheduleMetadataDurationHydration); mediaEl.removeEventListener("loadedmetadata", onMediaLoadedMetadataForProxy); mediaEl.removeEventListener("error", onMediaErrorForProxy); + mediaEl.removeEventListener("waiting", onMediaStallEvent); + mediaEl.removeEventListener("stalled", onMediaStallEvent); } metadataBoundMedia.clear(); }; @@ -1821,6 +1861,19 @@ export function initSandboxRuntimeModular(): void { const bindMediaMetadataListeners = () => { if (state.tornDown) return; const mediaEls = Array.from(document.querySelectorAll("video, audio")) as HTMLMediaElement[]; + // Segment-heavy compositions (editors emit one media element per cut) must + // not full-preload EVERYTHING at mount: dozens of parallel fetches contend + // for the connection pool and the EARLY segments buffer late. Above the + // threshold, far-future timed segments start metadata-only; the sync layer + // upgrades them to full preload as the playhead approaches (media.ts + // readiness stages set preload="auto" ~3s ahead). + const EAGER_PRELOAD_MAX_TIMED_MEDIA = 6; + const STAGED_PRELOAD_HORIZON_SECONDS = 10; + const timedMediaCount = mediaEls.reduce( + (count, el) => count + (el.hasAttribute("data-start") ? 1 : 0), + 0, + ); + const stagePreload = timedMediaCount > EAGER_PRELOAD_MAX_TIMED_MEDIA; for (const mediaEl of mediaEls) { if (metadataBoundMedia.has(mediaEl)) continue; metadataBoundMedia.add(mediaEl); @@ -1835,6 +1888,8 @@ export function initSandboxRuntimeModular(): void { // for