From 32a1492d05a6e17cb2d5f7d4676c2de92e29073b Mon Sep 17 00:00:00 2001 From: I-bangash Date: Thu, 30 Jul 2026 13:24:46 +0100 Subject: [PATCH 1/7] perf(core): cache-friendly audio fetch, edge gain ramps, fade-out stop in WebAudio transport - decodeAudioElement: first fetch per src uses the normal HTTP cache (media URLs are stable; edited assets ship under new URLs). no-store is applied only on a RETRY after a failed attempt, preserving the late-asset self-heal semantics from #1602 without re-downloading identical bytes on every fresh document. - schedulePlayback: ~8ms linear gain ramps at each source's effective start and (for bounded clips) into its scheduled end, so adjacent trimmed clips hand off with a micro-crossfade instead of a waveform-discontinuity pop. - stopAll: fade to zero and defer the stop ~20ms instead of hard-cutting mid-waveform; node teardown moves to the ended listener. --- .../src/runtime/webAudioTransport.test.ts | 116 +++++++++++++++- .../core/src/runtime/webAudioTransport.ts | 128 ++++++++++++++---- 2 files changed, 220 insertions(+), 24 deletions(-) diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index 15f1825a13..96e0da471e 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -17,7 +17,13 @@ function createMockAudioContext(currentTime = 100) { _fireEnded: () => endedListeners.forEach((cb) => cb()), }; const gainNode = { - gain: { value: 1 }, + gain: { + value: 1, + setValueAtTime: vi.fn(), + linearRampToValueAtTime: vi.fn(), + cancelScheduledValues: vi.fn(), + setTargetAtTime: vi.fn(), + }, connect: vi.fn(), disconnect: vi.fn(), }; @@ -464,5 +470,113 @@ describe("WebAudioTransport", () => { expect(fetchMock).toHaveBeenCalledTimes(1); // short-circuited, no re-fetch vi.unstubAllGlobals(); }); + + it("first fetch per src uses the HTTP cache; only a RETRY bypasses it with no-store", async () => { + const transport = transportWithDecode(async () => ({}) as AudioBuffer); + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ ok: false, status: 404 }) // transient — not blacklisted + .mockResolvedValueOnce({ ok: true, arrayBuffer: async () => new ArrayBuffer(8) }); + vi.stubGlobal("fetch", fetchMock); + + await transport.decodeAudioElement(el("tts.wav")); + // Media URLs are stable (edited assets ship under new URLs), so the first + // request must be allowed to hit the HTTP cache. + expect(fetchMock).toHaveBeenNthCalledWith(1, "tts.wav", undefined); + + await transport.decodeAudioElement(el("tts.wav")); + // A retry after a failure must actually re-request — not replay a cached + // 404 from the attempt we chose not to blacklist. + expect(fetchMock).toHaveBeenNthCalledWith(2, "tts.wav", { cache: "no-store" }); + vi.unstubAllGlobals(); + }); + }); + + describe("edge gain ramps (cut-boundary pop suppression)", () => { + async function rampsFor(args: { + compositionStart: number; + compositionTime: number; + volume: number; + clipDuration?: number; + }) { + const { transport, mock, gen } = setupTransport(100); + await transport.schedulePlayback( + mockEl, + mockBuffer, + args.compositionStart, + 0, + args.compositionTime, + args.volume, + gen, + 1, + args.clipDuration, + ); + return mock.gainNode.gain; + } + + function expectRamps( + g: Awaited>, + volume: number, + startAt: number, + endAt: number, + ) { + expect(g.setValueAtTime).toHaveBeenNthCalledWith(1, 0, startAt); + expect(g.linearRampToValueAtTime).toHaveBeenNthCalledWith(1, volume, startAt + 0.008); + expect(g.setValueAtTime).toHaveBeenNthCalledWith(2, volume, endAt - 0.008); + expect(g.linearRampToValueAtTime).toHaveBeenNthCalledWith(2, 0, endAt); + } + + it("ramps gain in at the source's effective start and out before a bounded end", async () => { + // elapsed = 8-5 = 3 into a 10s clip → starts now (100), ends at 107. + const g = await rampsFor({ + compositionStart: 5, + compositionTime: 8, + volume: 0.7, + clipDuration: 10, + }); + expectRamps(g, 0.7, 100, 107); + }); + + it("a future-scheduled bounded clip ramps at its delayed start/end", async () => { + // compositionStart 10 at compositionTime 8 → starts at ctx time 102, ends 112. + const g = await rampsFor({ + compositionStart: 10, + compositionTime: 8, + volume: 1, + clipDuration: 10, + }); + expectRamps(g, 1, 102, 112); + }); + + it("unbounded sources get only the ramp-in", async () => { + const g = await rampsFor({ compositionStart: 0, compositionTime: 0, volume: 1 }); + expect(g.setValueAtTime).toHaveBeenCalledTimes(1); + expect(g.linearRampToValueAtTime).toHaveBeenCalledTimes(1); + }); + }); + + describe("stopAll fade-out (pause/seek pop suppression)", () => { + it("fades gain and defers the stop instead of hard-cutting mid-waveform", async () => { + const { transport, mock, gen } = setupTransport(100); + const el = { muted: false } as HTMLMediaElement; + + await transport.schedulePlayback(el, mockBuffer, 0, 0, 0, 1, gen); + expect(el.muted).toBe(true); + + transport.stopAll(); + + const g = mock.gainNode.gain; + expect(g.cancelScheduledValues).toHaveBeenCalledWith(100); + expect(g.setTargetAtTime).toHaveBeenCalledWith(0, 100, expect.any(Number)); + expect(mock.sourceNode.stop).toHaveBeenCalledWith(expect.closeTo(100.02, 5)); + // State restores immediately; node teardown happens on `ended`. + expect(el.muted).toBe(false); + expect(transport.isActive()).toBe(false); + expect(mock.gainNode.disconnect).not.toHaveBeenCalled(); + + mock.sourceNode._fireEnded(); + expect(mock.sourceNode.disconnect).toHaveBeenCalled(); + expect(mock.gainNode.disconnect).toHaveBeenCalled(); + }); }); }); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index d4497ac389..40a2f779fc 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -70,10 +70,58 @@ export type ScheduledSource = { bounded: boolean; }; +/** Seconds of linear gain ramp applied at a scheduled source's start (and, for + * bounded clips, before its end). Short enough to be inaudible as a fade, long + * enough to remove the waveform-discontinuity click a hard cut produces. */ +const EDGE_RAMP_SECONDS = 0.008; + +/** Time constant for the stop-side fade in stopAll(); sources are stopped a few + * time constants later so pause/seek never end mid-waveform with a pop. */ +const STOP_FADE_TIME_CONSTANT = 0.004; +const STOP_FADE_TAIL_SECONDS = 0.02; + +/** + * Anti-pop gain automation: a hard cut starts/stops mid-waveform, which is an + * audible click. Ramp gain over EDGE_RAMP_SECONDS at the source's effective + * start and (for bounded clips) into its scheduled end, so adjacent trimmed + * clips hand off with a ~8ms micro-crossfade instead of a pop. Best-effort: a + * missing/throwing AudioParam API must never break playback. + */ +function applyEdgeGainRamps( + gain: GainNode["gain"], + opts: { + volume: number; + elapsed: number; + scheduledAt: number; + safeRate: number; + clipDuration: number; + }, +): void { + try { + const { volume, elapsed, scheduledAt, safeRate, clipDuration } = opts; + const startAt = elapsed >= 0 ? scheduledAt : scheduledAt + -elapsed / safeRate; + gain.setValueAtTime(0, startAt); + gain.linearRampToValueAtTime(volume, startAt + EDGE_RAMP_SECONDS); + const hasBound = Number.isFinite(clipDuration) && clipDuration > 0; + if (!hasBound) return; + const clipSourceLen = clipDuration * safeRate; + const endAt = + elapsed >= 0 + ? scheduledAt + (clipSourceLen - elapsed) / safeRate + : startAt + clipSourceLen / safeRate; + if (endAt - startAt <= EDGE_RAMP_SECONDS * 4) return; + gain.setValueAtTime(volume, endAt - EDGE_RAMP_SECONDS); + gain.linearRampToValueAtTime(0, endAt); + } catch (err) { + swallow("webAudioTransport.edgeRamp", err); + } +} + export class WebAudioTransport { private _ctx: AudioContext | null = null; private _bufferCache = new Map(); private _failedSrcs = new Set(); + private _fetchAttemptedSrcs = new Set(); private _activeSources: ScheduledSource[] = []; private _masterGain: GainNode | null = null; // Composition-time reference frame: at AudioContext time `_rateAnchorCtx`, @@ -112,26 +160,8 @@ export class WebAudioTransport { if (this._failedSrcs.has(src)) return null; if (!this._ctx) return null; - // Fetch the bytes. A network error or non-OK status (e.g. a 404 for an - // asset that simply has not been uploaded yet) is TRANSIENT — return null - // WITHOUT blacklisting, so the next play/seek generation retries once the - // asset becomes available. (Previously these were added to `_failedSrcs`, - // which is never cleared, permanently silencing a merely-late track.) - let arrayBuffer: ArrayBuffer; - try { - // `no-store`: a retry must actually re-request the asset — not replay a - // cached 404/stale response from the failed attempt that we chose not to - // blacklist. - const response = await fetch(src, { cache: "no-store" }); - if (!response.ok) { - swallow("webAudioTransport.fetch", new Error(`${response.status} ${src}`)); - return null; - } - arrayBuffer = await response.arrayBuffer(); - } catch (err) { - swallow("webAudioTransport.fetch", err); - return null; - } + const arrayBuffer = await this._fetchAudioBytes(src); + if (!arrayBuffer) return null; // A decode failure means the bytes themselves are unusable (corrupt or an // unsupported codec) — that IS permanent, so blacklist to avoid re-decoding @@ -147,6 +177,35 @@ export class WebAudioTransport { } } + /** + * Fetch the bytes for a source. A network error or non-OK status (e.g. a 404 + * for an asset that simply has not been uploaded yet) is TRANSIENT — return + * null WITHOUT blacklisting, so the next play/seek generation retries once + * the asset becomes available. (Previously these were added to `_failedSrcs`, + * which is never cleared, permanently silencing a merely-late track.) + * + * The first attempt per src uses the normal HTTP cache — media sources are + * stable URLs (an edited asset ships under a new URL), so bypassing the cache + * here just re-downloads identical bytes on every fresh document. `no-store` + * only on a RETRY: a retry must actually re-request the asset — not replay a + * cached 404/stale response from the failed attempt we chose not to blacklist. + */ + private async _fetchAudioBytes(src: string): Promise { + try { + const isRetry = this._fetchAttemptedSrcs.has(src); + this._fetchAttemptedSrcs.add(src); + const response = await fetch(src, isRetry ? { cache: "no-store" } : undefined); + if (!response.ok) { + swallow("webAudioTransport.fetch", new Error(`${response.status} ${src}`)); + return null; + } + return await response.arrayBuffer(); + } catch (err) { + swallow("webAudioTransport.fetch", err); + return null; + } + } + startGeneration(): number { this._playGeneration += 1; return this._playGeneration; @@ -208,6 +267,8 @@ export class WebAudioTransport { return null; } + applyEdgeGainRamps(gainNode.gain, { volume, elapsed, scheduledAt, safeRate, clipDuration }); + const priorMuted = el.muted; el.muted = true; logFallbackHandoff(el, priorMuted); @@ -226,6 +287,14 @@ export class WebAudioTransport { this._paused = false; sourceNode.addEventListener("ended", () => { + // Sources are one-shot; release the graph nodes whether the source + // ended naturally or via stopAll()'s deferred fade-out stop. + try { + sourceNode.disconnect(); + gainNode.disconnect(); + } catch { + // already disconnected + } const idx = this._activeSources.indexOf(scheduled); if (idx !== -1) { this._activeSources.splice(idx, 1); @@ -273,11 +342,24 @@ export class WebAudioTransport { } stopAll(): void { + const ctx = this._ctx; for (const source of this._activeSources) { try { - source.sourceNode.stop(); - source.sourceNode.disconnect(); - source.gainNode.disconnect(); + if (ctx) { + // Fade instead of a hard cut so pause/seek never stop mid-waveform + // with an audible pop. The source rings for ~STOP_FADE_TAIL_SECONDS + // at rapidly-decaying gain, then stops; its `ended` listener + // disconnects the nodes. (A seek's freshly-scheduled sources overlap + // this brief tail — a micro-crossfade, which is the point.) + const now = ctx.currentTime; + source.gainNode.gain.cancelScheduledValues(now); + source.gainNode.gain.setTargetAtTime(0, now, STOP_FADE_TIME_CONSTANT); + source.sourceNode.stop(now + STOP_FADE_TAIL_SECONDS); + } else { + source.sourceNode.stop(); + source.sourceNode.disconnect(); + source.gainNode.disconnect(); + } } catch { // already stopped } From f75042a440d95b944704ae8a4ed43a790442829e Mon Sep 17 00:00:00 2001 From: I-bangash Date: Thu, 30 Jul 2026 13:25:18 +0100 Subject: [PATCH 2/7] perf(core): pre-decode audio at mount, upcoming-clip readiness, stall telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cut-boundary playback smoothness for segment-heavy compositions (an editor emitting one media element per cut): - init: decode-only warm pass over audio[data-start] as soon as the WebAudio transport exists (and periodically for late-loaded sub-compositions), so first play after a document (re)load doesn't run full-file decodeAudioData lazily while sound falls back to the buffering HTMLMedia element. - init: staged preload — when a composition carries >6 timed media elements, far-future segments (start >10s) begin preload=metadata; the sync layer upgrades them as the playhead approaches. Stops the mount-time fetch stampede that leaves EARLY segments buffering late. - media: two readiness stages for upcoming clips while playing: pre-seek to the clip's media offset ~3s ahead (buffer the RIGHT region), and a muted video pre-roll ~350ms ahead so activation is a visibility flip instead of a cold seek+play (decoder reset ~150ms freeze). - init: bounded runtime_media_stall diagnostics (waiting/stalled while playing) + freeze the transport clock when a buffering ACTIVE video has no audio mastering the clock (video-only compositions), mirroring the existing audio-buffering freeze. --- packages/core/src/runtime/init.test.ts | 57 ++++++++++++ packages/core/src/runtime/init.ts | 114 +++++++++++++++++++++++- packages/core/src/runtime/media.test.ts | 70 +++++++++++++++ packages/core/src/runtime/media.ts | 87 ++++++++++++++++++ 4 files changed, 326 insertions(+), 2 deletions(-) diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index ff91e738ed..9f81d9de71 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -2683,4 +2683,61 @@ describe("initSandboxRuntimeModular", () => { }).not.toThrow(); }); }); + + describe("audio buffer pre-decode at mount", () => { + it("warm-decodes every timed audio source before any play()", async () => { + const decodeAudioData = vi.fn(async () => ({}) as AudioBuffer); + class MockAudioContext { + currentTime = 0; + state = "running"; + destination = {}; + decodeAudioData = decodeAudioData; + createGain() { + return { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() }; + } + resume = vi.fn(); + close = vi.fn(); + } + vi.stubGlobal("AudioContext", MockAudioContext); + const fetchMock = vi.fn(async () => ({ + ok: true, + arrayBuffer: async () => new ArrayBuffer(8), + })); + vi.stubGlobal("fetch", fetchMock); + + try { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-duration", "10"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + const audioA = document.createElement("audio"); + audioA.setAttribute("data-start", "0"); + audioA.setAttribute("data-duration", "5"); + audioA.setAttribute("src", "https://media.example/a.mp3"); + const audioB = document.createElement("audio"); + audioB.setAttribute("data-start", "5"); + audioB.setAttribute("data-duration", "5"); + audioB.setAttribute("src", "https://media.example/b.mp3"); + root.append(audioA, audioB); + document.body.appendChild(root); + window.__timelines = { main: createMockTimeline(10) }; + + initSandboxRuntimeModular(); + + // No play() has happened — the decode must be driven by mount alone. + await vi.waitFor(() => { + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(decodeAudioData).toHaveBeenCalledTimes(2); + }); + const urls = fetchMock.mock.calls.map((call) => String(call[0])); + expect(urls.some((u) => u.endsWith("a.mp3"))).toBe(true); + expect(urls.some((u) => u.endsWith("b.mp3"))).toBe(true); + } finally { + vi.unstubAllGlobals(); + } + }); + }); }); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index cf65ecd262..d6b268c2c0 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -174,6 +174,9 @@ export function initSandboxRuntimeModular(): void { let webAudioReady = false; void webAudio.init().then((ok) => { webAudioReady = ok; + // Warm the decoded-buffer cache as soon as the transport exists (decode + // works on a suspended AudioContext, so no user gesture is needed). + if (ok) predecodeAudioClipBuffers(); }); // `_auto` is a Studio-internal keyframe marker (an auto-tracked endpoint the // parser reads back), NOT an animatable property. Register it as a no-op GSAP @@ -1808,12 +1811,49 @@ export function initSandboxRuntimeModular(): void { } }; + // Boundary-readiness telemetry: surface playback stalls (`waiting`/`stalled` + // on a timed media element while the transport plays) so cut-boundary tuning + // can target the right layer. Bounded per document to avoid diagnostic spam. + let mediaStallDiagnosticsPosted = 0; + const MAX_MEDIA_STALL_DIAGNOSTICS = 20; + const onMediaStallEvent = (event: Event) => { + const el = event.currentTarget; + if (!(el instanceof HTMLMediaElement)) return; + if (!state.isPlaying || state.tornDown) return; + if (mediaStallDiagnosticsPosted >= MAX_MEDIA_STALL_DIAGNOSTICS) return; + mediaStallDiagnosticsPosted += 1; + let bufferedEnd: number | null = null; + try { + bufferedEnd = el.buffered.length > 0 ? el.buffered.end(el.buffered.length - 1) : null; + } catch { + // buffered ranges unavailable + } + postRuntimeMessage({ + source: "hf-preview", + type: "diagnostic", + code: "runtime_media_stall", + details: { + event: event.type, + tagName: el.tagName.toLowerCase(), + currentSrc: el.currentSrc || null, + readyState: el.readyState, + networkState: el.networkState, + mediaTime: el.currentTime, + bufferedEnd, + compositionTime: state.currentTime, + clipStart: Number.parseFloat(el.dataset.start ?? "") || null, + }, + }); + }; + const unbindMediaMetadataListeners = () => { for (const mediaEl of metadataBoundMedia) { mediaEl.removeEventListener("loadedmetadata", scheduleMetadataDurationHydration); mediaEl.removeEventListener("durationchange", scheduleMetadataDurationHydration); mediaEl.removeEventListener("loadedmetadata", onMediaLoadedMetadataForProxy); mediaEl.removeEventListener("error", onMediaErrorForProxy); + mediaEl.removeEventListener("waiting", onMediaStallEvent); + mediaEl.removeEventListener("stalled", onMediaStallEvent); } metadataBoundMedia.clear(); }; @@ -1821,6 +1861,19 @@ export function initSandboxRuntimeModular(): void { const bindMediaMetadataListeners = () => { if (state.tornDown) return; const mediaEls = Array.from(document.querySelectorAll("video, audio")) as HTMLMediaElement[]; + // Segment-heavy compositions (editors emit one media element per cut) must + // not full-preload EVERYTHING at mount: dozens of parallel fetches contend + // for the connection pool and the EARLY segments buffer late. Above the + // threshold, far-future timed segments start metadata-only; the sync layer + // upgrades them to full preload as the playhead approaches (media.ts + // readiness stages set preload="auto" ~3s ahead). + const EAGER_PRELOAD_MAX_TIMED_MEDIA = 6; + const STAGED_PRELOAD_HORIZON_SECONDS = 10; + const timedMediaCount = mediaEls.reduce( + (count, el) => count + (el.hasAttribute("data-start") ? 1 : 0), + 0, + ); + const stagePreload = timedMediaCount > EAGER_PRELOAD_MAX_TIMED_MEDIA; for (const mediaEl of mediaEls) { if (metadataBoundMedia.has(mediaEl)) continue; metadataBoundMedia.add(mediaEl); @@ -1835,6 +1888,8 @@ export function initSandboxRuntimeModular(): void { // for