perf(core): smooth cut-boundary playback — mount-time audio pre-decode, upcoming-clip readiness, edge gain ramps

packages/core/src/runtime/init.test.ts

@@ -2683,4 +2683,128 @@ describe("initSandboxRuntimeModular", () => {
       }).not.toThrow();
     });
   });
+
+  describe("audio buffer pre-decode at mount", () => {
+    it("warm-decodes every timed audio source before any play()", async () => {
+      const decodeAudioData = vi.fn(async () => ({}) as AudioBuffer);
+      class MockAudioContext {
+        currentTime = 0;
+        state = "running";
+        destination = {};
+        decodeAudioData = decodeAudioData;
+        createGain() {
+          return { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() };
+        }
+        resume = vi.fn();
+        close = vi.fn();
+      }
+      vi.stubGlobal("AudioContext", MockAudioContext);
+      const fetchMock = vi.fn(async () => ({
+        ok: true,
+        arrayBuffer: async () => new ArrayBuffer(8),
+      }));
+      vi.stubGlobal("fetch", fetchMock);
+
+      try {
+        const root = document.createElement("div");
+        root.setAttribute("data-composition-id", "main");
+        root.setAttribute("data-root", "true");
+        root.setAttribute("data-start", "0");
+        root.setAttribute("data-duration", "10");
+        root.setAttribute("data-width", "1920");
+        root.setAttribute("data-height", "1080");
+        const audioA = document.createElement("audio");
+        audioA.setAttribute("data-start", "0");
+        audioA.setAttribute("data-duration", "5");
+        audioA.setAttribute("src", "https://media.example/a.mp3");
+        const audioB = document.createElement("audio");
+        audioB.setAttribute("data-start", "5");
+        audioB.setAttribute("data-duration", "5");
+        audioB.setAttribute("src", "https://media.example/b.mp3");
+        root.append(audioA, audioB);
+        document.body.appendChild(root);
+        window.__timelines = { main: createMockTimeline(10) };
+
+        initSandboxRuntimeModular();
+
+        // No play() has happened — the decode must be driven by mount alone.
+        await vi.waitFor(() => {
+          expect(fetchMock).toHaveBeenCalledTimes(2);
+          expect(decodeAudioData).toHaveBeenCalledTimes(2);
+        });
+        const urls = fetchMock.mock.calls.map((call) => String(call[0]));
+        expect(urls.some((u) => u.endsWith("a.mp3"))).toBe(true);
+        expect(urls.some((u) => u.endsWith("b.mp3"))).toBe(true);
+      } finally {
+        vi.unstubAllGlobals();
+      }
+    });
+
+    it("only warms clips near the playhead, and honors the embedder opt-out", async () => {
+      const decodeAudioData = vi.fn(async () => ({}) as AudioBuffer);
+      class MockAudioContext {
+        currentTime = 0;
+        state = "running";
+        destination = {};
+        decodeAudioData = decodeAudioData;
+        createGain() {
+          return { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() };
+        }
+        resume = vi.fn();
+        close = vi.fn();
+      }
+      vi.stubGlobal("AudioContext", MockAudioContext);
+      const fetchMock = vi.fn(async () => ({
+        ok: true,
+        arrayBuffer: async () => new ArrayBuffer(8),
+      }));
+      vi.stubGlobal("fetch", fetchMock);
+
+      try {
+        const root = document.createElement("div");
+        root.setAttribute("data-composition-id", "main");
+        root.setAttribute("data-root", "true");
+        root.setAttribute("data-start", "0");
+        root.setAttribute("data-duration", "600");
+        root.setAttribute("data-width", "1920");
+        root.setAttribute("data-height", "1080");
+        const near = document.createElement("audio");
+        near.setAttribute("data-start", "0");
+        near.setAttribute("data-duration", "5");
+        near.setAttribute("src", "https://media.example/near.mp3");
+        const far = document.createElement("audio");
+        far.setAttribute("data-start", "300");
+        far.setAttribute("data-duration", "5");
+        far.setAttribute("src", "https://media.example/far.mp3");
+        root.append(near, far);
+        document.body.appendChild(root);
+        window.__timelines = { main: createMockTimeline(600) };
+
+        initSandboxRuntimeModular();
+
+        // Warming a whole long composition costs a full-file fetch + ~23MB of
+        // PCM per stereo minute PER SOURCE — only playhead-proximate clips may
+        // pre-decode; the rest stay lazy until the playhead approaches.
+        await vi.waitFor(() => {
+          expect(fetchMock).toHaveBeenCalledTimes(1);
+        });
+        expect(String(fetchMock.mock.calls[0]?.[0])).toContain("near.mp3");
+
+        window.__hfRuntimeTeardown?.();
+        fetchMock.mockClear();
+        (
+          window as Window & { __HF_AUDIO_PREDECODE_DISABLED?: boolean }
+        ).__HF_AUDIO_PREDECODE_DISABLED = true;
+        initSandboxRuntimeModular();
+        // A document that exists but is never played (e.g. a double-buffered
+        // preview's standby iframe) must not warm anything.
+        await new Promise((r) => setTimeout(r, 20));
+        expect(fetchMock).not.toHaveBeenCalled();
+      } finally {
+        delete (window as Window & { __HF_AUDIO_PREDECODE_DISABLED?: boolean })
+          .__HF_AUDIO_PREDECODE_DISABLED;
+        vi.unstubAllGlobals();
+      }
+    });
+  });
 });

packages/core/src/runtime/init.ts

@@ -174,6 +174,9 @@ export function initSandboxRuntimeModular(): void {
   let webAudioReady = false;
   void webAudio.init().then((ok) => {
     webAudioReady = ok;
+    // Warm the decoded-buffer cache as soon as the transport exists (decode
+    // works on a suspended AudioContext, so no user gesture is needed).
+    if (ok) predecodeAudioClipBuffers();
   });
   // `_auto` is a Studio-internal keyframe marker (an auto-tracked endpoint the
   // parser reads back), NOT an animatable property. Register it as a no-op GSAP
@@ -1808,19 +1811,69 @@ export function initSandboxRuntimeModular(): void {
     }
   };
 
+  // Boundary-readiness telemetry: surface playback stalls (`waiting`/`stalled`
+  // on a timed media element while the transport plays) so cut-boundary tuning
+  // can target the right layer. Bounded per document to avoid diagnostic spam.
+  let mediaStallDiagnosticsPosted = 0;
+  const MAX_MEDIA_STALL_DIAGNOSTICS = 20;
+  const onMediaStallEvent = (event: Event) => {
+    const el = event.currentTarget;
+    if (!(el instanceof HTMLMediaElement)) return;
+    if (!state.isPlaying || state.tornDown) return;
+    if (mediaStallDiagnosticsPosted >= MAX_MEDIA_STALL_DIAGNOSTICS) return;
+    mediaStallDiagnosticsPosted += 1;
+    let bufferedEnd: number | null = null;
+    try {
+      bufferedEnd = el.buffered.length > 0 ? el.buffered.end(el.buffered.length - 1) : null;
+    } catch {
+      // buffered ranges unavailable
+    }
+    postRuntimeMessage({
+      source: "hf-preview",
+      type: "diagnostic",
+      code: "runtime_media_stall",
+      details: {
+        event: event.type,
+        tagName: el.tagName.toLowerCase(),
+        currentSrc: el.currentSrc || null,
+        readyState: el.readyState,
+        networkState: el.networkState,
+        mediaTime: el.currentTime,
+        bufferedEnd,
+        compositionTime: state.currentTime,
+        clipStart: Number.parseFloat(el.dataset.start ?? "") || null,
+      },
+    });
+  };
+
   const unbindMediaMetadataListeners = () => {
     for (const mediaEl of metadataBoundMedia) {
       mediaEl.removeEventListener("loadedmetadata", scheduleMetadataDurationHydration);
       mediaEl.removeEventListener("durationchange", scheduleMetadataDurationHydration);
       mediaEl.removeEventListener("loadedmetadata", onMediaLoadedMetadataForProxy);
       mediaEl.removeEventListener("error", onMediaErrorForProxy);
+      mediaEl.removeEventListener("waiting", onMediaStallEvent);
+      mediaEl.removeEventListener("stalled", onMediaStallEvent);
     }
     metadataBoundMedia.clear();
   };
 
   const bindMediaMetadataListeners = () => {
     if (state.tornDown) return;
     const mediaEls = Array.from(document.querySelectorAll("video, audio")) as HTMLMediaElement[];
+    // Segment-heavy compositions (editors emit one media element per cut) must
+    // not full-preload EVERYTHING at mount: dozens of parallel fetches contend
+    // for the connection pool and the EARLY segments buffer late. Above the
+    // threshold, far-future timed segments start metadata-only; the sync layer
+    // upgrades them to full preload as the playhead approaches (media.ts
+    // readiness stages set preload="auto" ~3s ahead).
+    const EAGER_PRELOAD_MAX_TIMED_MEDIA = 6;
+    const STAGED_PRELOAD_HORIZON_SECONDS = 10;
+    const timedMediaCount = mediaEls.reduce(
+      (count, el) => count + (el.hasAttribute("data-start") ? 1 : 0),
+      0,
+    );
+    const stagePreload = timedMediaCount > EAGER_PRELOAD_MAX_TIMED_MEDIA;
     for (const mediaEl of mediaEls) {
       if (metadataBoundMedia.has(mediaEl)) continue;
       metadataBoundMedia.add(mediaEl);
@@ -1835,6 +1888,8 @@ export function initSandboxRuntimeModular(): void {
       // for <audio> — all guarded inside mediaProxy.ts itself.
       mediaEl.addEventListener("loadedmetadata", onMediaLoadedMetadataForProxy);
       mediaEl.addEventListener("error", onMediaErrorForProxy);
+      mediaEl.addEventListener("waiting", onMediaStallEvent);
+      mediaEl.addEventListener("stalled", onMediaStallEvent);
 
       // Proactive proxy-fallback trigger: consult the codec map and swap
       // BEFORE the eager load() below, so a known-hostile asset never even
@@ -1845,8 +1900,12 @@ export function initSandboxRuntimeModular(): void {
       // Eagerly preload media data so audio/video is buffered before the user
       // clicks play. Without this, the first play() call fires on un-fetched
       // media, producing silence or choppy audio until the browser caches it.
-      if (mediaEl.preload !== "auto") {
-        mediaEl.preload = "auto";
+      // Exception: staged preload for far-future segments (see above).
+      const startAttr = Number.parseFloat(mediaEl.dataset.start ?? "");
+      const farFuture = Number.isFinite(startAttr) && startAttr > STAGED_PRELOAD_HORIZON_SECONDS;
+      const targetPreload = stagePreload && farFuture ? "metadata" : "auto";
+      if (mediaEl.preload !== targetPreload) {
+        mediaEl.preload = targetPreload;
       }
       if (mediaEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
         mediaEl.load();
@@ -1861,6 +1920,53 @@ export function initSandboxRuntimeModular(): void {
     }
   };
 
+  // Decode-only warm pass over timed audio sources NEAR THE PLAYHEAD so
+  // WebAudio buffers are ready BEFORE the first play() instead of being decoded
+  // lazily inside it. The lazy path costs a full-file fetch+decodeAudioData per
+  // source at the moment the user hits play — on a freshly (re)loaded document
+  // (every editor edit reloads the preview iframe) that gap plays through the
+  // HTMLMedia fallback, heard as an audio dropout. Pre-decoding overlaps that
+  // work with document load.
+  //
+  // BOUNDED, not exhaustive: a source's fetch pulls the WHOLE file into memory
+  // and the decoded PCM is ~23MB per stereo minute — warming EVERY clip of a
+  // long multi-clip composition on EVERY document load multiplies to gigabytes
+  // (an editor with a standby iframe doubles it again). So each pass only warms
+  // clips whose window is near the current time; the periodic re-run (every 30
+  // transport ticks) slides the window as the playhead moves, and play() still
+  // decodes whatever it needs exactly as before. `decodeAudioElement` dedupes
+  // via its buffer cache and failure blacklist, so repeated calls per element
+  // are cheap map hits.
+  //
+  // Skipped during render capture (the producer mixes audio with ffmpeg) and
+  // when the embedder set `__HF_AUDIO_PREDECODE_DISABLED` — the opt-out for
+  // documents that exist but are never played (e.g. a double-buffered editor
+  // preview's hidden standby iframe).
+  const PREDECODE_BEHIND_SECONDS = 10;
+  const PREDECODE_AHEAD_SECONDS = 45;
+  const predecodeAudioClipBuffers = () => {
+    if (state.tornDown || !webAudioReady) return;
+    if (state.nativeMediaSyncDisabled || state.webAudioMediaDisabled) return;
+    const w = window as Window & {
+      __HF_RENDER_CAPTURE_MODE?: boolean;
+      __HF_AUDIO_PREDECODE_DISABLED?: boolean;
+    };
+    if (w.__HF_RENDER_CAPTURE_MODE || w.__HF_AUDIO_PREDECODE_DISABLED) return;
+    const now = Math.max(0, state.currentTime || 0);
+    const windowStart = now - PREDECODE_BEHIND_SECONDS;
+    const windowEnd = now + PREDECODE_AHEAD_SECONDS;
+    const audioEls = document.querySelectorAll("audio[data-start]");
+    for (const el of audioEls) {
+      if (!(el instanceof HTMLMediaElement) || !el.isConnected) continue;
+      const start = Number.parseFloat(el.dataset.start ?? "");
+      if (!Number.isFinite(start)) continue;
+      const durAttr = Number.parseFloat(el.dataset.duration ?? "");
+      const end = Number.isFinite(durAttr) && durAttr > 0 ? start + durAttr : Infinity;
+      if (end < windowStart || start > windowEnd) continue;
+      void webAudio.decodeAudioElement(el);
+    }
+  };
+
   const probeAndCacheVolumeKeyframes = (mediaEl: HTMLMediaElement) => {
     if (volumeKeyframeCache.has(mediaEl)) return;
     probeAndCacheElementVolume(
@@ -2882,6 +2988,9 @@ export function initSandboxRuntimeModular(): void {
       }
       if (transportTickCount % 30 === 0) {
         bindMediaMetadataListeners();
+        // Also warm decode for audio elements discovered after mount (loaded
+        // sub-compositions) or bound before the async WebAudio init resolved.
+        predecodeAudioClipBuffers();
       }
 
       // Sync clock duration with the resolved timeline each tick (catches async
@@ -2933,6 +3042,15 @@ export function initSandboxRuntimeModular(): void {
               break;
             }
           }
+          // NOTE: an earlier revision also froze the clock here while an ACTIVE
+          // video was buffering (mirroring the audio-buffering freeze above).
+          // That proved hazardous: when the browser silently EVICTS a media
+          // element's resource under memory pressure (readyState collapses to
+          // HAVE_NOTHING with no error and, without recovery, never rises
+          // again), the frozen clock never released — a permanent playback
+          // hang, worse than the transient video lag it was smoothing. Video
+          // stalls now surface via the runtime_media_stall diagnostics and the
+          // eviction-recovery reload in syncRuntimeMedia instead.
           if (!foundActive && clock.hasAudioSource()) {
             clock.detachAudioSource();
           }
@@ -3247,6 +3365,7 @@ export function initSandboxRuntimeModular(): void {
       window.removeEventListener("beforeunload", state.beforeUnloadHandler);
       state.beforeUnloadHandler = null;
     }
+    window.removeEventListener("pagehide", teardown);
     picker.disablePickMode();
     for (const adapter of state.deterministicAdapters) {
       if (!adapter || typeof adapter.revert !== "function") continue;
@@ -3301,4 +3420,12 @@ export function initSandboxRuntimeModular(): void {
   window.__hfRuntimeTeardown = teardown;
   state.beforeUnloadHandler = teardown;
   window.addEventListener("beforeunload", state.beforeUnloadHandler);
+  // `pagehide` too: in an iframe whose document is replaced (an editor writing
+  // a new `srcdoc` on every edit), pagehide is the dependable unload signal.
+  // Without a teardown there, each discarded document keeps a live
+  // AudioContext + decoded-buffer cache until GC gets around to it — under
+  // rapid edits that transiently stacks hundreds of MB of PCM per reload.
+  // teardown() is idempotent (state.tornDown), so double-firing with
+  // beforeunload is harmless.
+  window.addEventListener("pagehide", teardown);
 }