SMOODEV-2668: Fix browser-voice TTS static (frame alignment + proper resampling)

.changeset/voice-static-fix.md

@@ -0,0 +1,5 @@
+---
+'@smooai/chat-widget': patch
+---
+
+Fix heavy static on browser-voice TTS playback (SMOODEV-2668): re-align linear16 frames that split mid-sample (an odd-length frame made every later frame decode one byte off — pure noise), resample 16 kHz audio to the AudioContext's native rate with a proper band-limited streaming upsampler instead of forcing a 16 kHz context (whose browser-side output resampling mirrored speech energy above 8 kHz as harsh imaging static), and prime playback ~100 ms off the playhead when the queue drains so just-in-time chunks don't click.

src/voice-session.test.ts

@@ -16,8 +16,10 @@ import { describe, expect, it } from 'vitest';
 import {
     downsampleTo16k,
     PcmPlayer,
+    PLAYBACK_PRIME_S,
     type PlayerAudioContext,
     rmsLevel,
+    StreamUpsampler,
     VOICE_SAMPLE_RATE,
     type VoicePlayer,
     VoiceSession,
@@ -364,17 +366,29 @@ interface ScheduledSource {
     length: number;
 }
 
-/** Fake 16 kHz AudioContext capturing scheduling for assertions. */
-function makeFakeCtx(): { ctx: PlayerAudioContext; scheduled: ScheduledSource[]; setTime: (t: number) => void; closed: () => boolean } {
+/** Fake AudioContext capturing scheduling + written samples for assertions. */
+function makeFakeCtx(sampleRate = 16000): {
+    ctx: PlayerAudioContext;
+    scheduled: ScheduledSource[];
+    buffers: Float32Array[];
+    setTime: (t: number) => void;
+    closed: () => boolean;
+} {
     let now = 0;
     let closed = false;
     const scheduled: ScheduledSource[] = [];
+    const buffers: Float32Array[] = [];
     const ctx: PlayerAudioContext = {
         get currentTime() {
             return now;
         },
         destination: {},
-        createBuffer: (_ch, length) => ({ getChannelData: () => new Float32Array(length) }),
+        sampleRate,
+        createBuffer: (_ch, length) => {
+            const data = new Float32Array(length);
+            buffers.push(data);
+            return { getChannelData: () => data };
+        },
         createBufferSource: () => {
             const rec: ScheduledSource = { startedAt: undefined, stopped: false, length: 0 };
             scheduled.push(rec);
@@ -395,31 +409,31 @@ function makeFakeCtx(): { ctx: PlayerAudioContext; scheduled: ScheduledSource[];
             return Promise.resolve();
         },
     };
-    return { ctx, scheduled, setTime: (t) => (now = t), closed: () => closed };
+    return { ctx, scheduled, buffers, setTime: (t) => (now = t), closed: () => closed };
 }
 
 describe('PcmPlayer', () => {
     const chunk = (samples: number): ArrayBuffer => new Int16Array(samples).buffer as ArrayBuffer;
 
-    it('schedules consecutive chunks back-to-back (gapless)', () => {
+    it('schedules consecutive chunks back-to-back (gapless), primed off the playhead', () => {
         const { ctx, scheduled } = makeFakeCtx();
         const player = new PcmPlayer(ctx);
         player.enqueue(chunk(1600)); // 100 ms
         player.enqueue(chunk(800)); // 50 ms
         player.enqueue(chunk(1600));
         const starts = scheduled.map((s) => s.startedAt!);
-        expect(starts[0]).toBe(0);
-        expect(starts[1]).toBeCloseTo(0.1, 10);
-        expect(starts[2]).toBeCloseTo(0.15, 10);
+        expect(starts[0]).toBe(PLAYBACK_PRIME_S);
+        expect(starts[1]).toBeCloseTo(PLAYBACK_PRIME_S + 0.1, 10);
+        expect(starts[2]).toBeCloseTo(PLAYBACK_PRIME_S + 0.15, 10);
     });
 
-    it('re-anchors to currentTime after the queue drains (no scheduling in the past)', () => {
+    it('re-anchors (primed) after the queue drains — no scheduling in the past', () => {
         const { ctx, scheduled, setTime } = makeFakeCtx();
         const player = new PcmPlayer(ctx);
-        player.enqueue(chunk(1600)); // plays 0 → 0.1
+        player.enqueue(chunk(1600)); // plays PRIME → PRIME+0.1
         setTime(0.5); // long silence; next reply arrives later
         player.enqueue(chunk(1600));
-        expect(scheduled[1]!.startedAt).toBe(0.5);
+        expect(scheduled[1]!.startedAt).toBeCloseTo(0.5 + PLAYBACK_PRIME_S, 10);
     });
 
     it('flush() stops every live source and resets the schedule', () => {
@@ -432,7 +446,65 @@ describe('PcmPlayer', () => {
         // After a flush the next chunk anchors to now, not the stale nextTime.
         setTime(0.05);
         player.enqueue(chunk(1600));
-        expect(scheduled[2]!.startedAt).toBe(0.05);
+        expect(scheduled[2]!.startedAt).toBeCloseTo(0.05 + PLAYBACK_PRIME_S, 10);
+    });
+
+    it('reassembles samples split mid-sample across frames (odd-length chunks)', () => {
+        // The wire is one continuous byte stream; a frame boundary can land in
+        // the middle of an int16. Split [100, -200, 300, 400] as 3+5 bytes.
+        const { ctx, buffers } = makeFakeCtx();
+        const player = new PcmPlayer(ctx);
+        const stream = new Uint8Array(new Int16Array([100, -200, 300, 400]).buffer);
+        player.enqueue(stream.slice(0, 3).buffer as ArrayBuffer);
+        player.enqueue(stream.slice(3).buffer as ArrayBuffer);
+        const played = buffers.flatMap((b) => Array.from(b));
+        expect(played).toEqual([100 / 32768, -200 / 32768, 300 / 32768, 400 / 32768]);
+    });
+
+    it('upsamples to a non-16k context rate and advances the schedule in output time', () => {
+        const { ctx, scheduled, buffers } = makeFakeCtx(48000);
+        const player = new PcmPlayer(ctx);
+        player.enqueue(chunk(1600)); // 100 ms of input
+        const outLen = buffers[0]!.length;
+        // ~3× the input samples (minus the sinc kernel's startup history).
+        expect(outLen).toBeGreaterThan(4600);
+        expect(outLen).toBeLessThanOrEqual(4800);
+        expect(scheduled[0]!.startedAt).toBe(PLAYBACK_PRIME_S);
+        // nextTime advanced by outLen/48000: the follow-up chunk butts against it.
+        player.enqueue(chunk(1600));
+        expect(scheduled[1]!.startedAt).toBeCloseTo(PLAYBACK_PRIME_S + outLen / 48000, 10);
+    });
+
+    it('upsampler output is identical whether the stream is split or not (no boundary artifacts)', () => {
+        // 440 Hz tone at 16 kHz, processed whole vs. split at awkward points —
+        // the carried filter state must make the outputs bit-identical.
+        const input = new Float32Array(1600);
+        for (let i = 0; i < input.length; i++) input[i] = Math.sin((2 * Math.PI * 440 * i) / VOICE_SAMPLE_RATE);
+        const whole = new StreamUpsampler(VOICE_SAMPLE_RATE, 48000).process(input);
+        const split = new StreamUpsampler(VOICE_SAMPLE_RATE, 48000);
+        const parts = [input.slice(0, 7), input.slice(7, 700), input.slice(700, 701), input.slice(701)].map((p) => split.process(p));
+        const joined = parts.flatMap((p) => Array.from(p));
+        expect(joined.length).toBe(whole.length);
+        for (let i = 0; i < whole.length; i++) {
+            expect(Math.abs(joined[i]! - whole[i]!)).toBeLessThan(1e-9);
+        }
+    });
+
+    it('upsampler reproduces an in-band tone faithfully (no imaging energy)', () => {
+        // A 1 kHz tone upsampled 16k → 48k should still be a clean 1 kHz tone:
+        // compare against the analytically expected samples (skip the kernel
+        // warm-up at the head).
+        const input = new Float32Array(3200);
+        for (let i = 0; i < input.length; i++) input[i] = Math.sin((2 * Math.PI * 1000 * i) / VOICE_SAMPLE_RATE);
+        const up = new StreamUpsampler(VOICE_SAMPLE_RATE, 48000);
+        const out = up.process(input);
+        const delaySamples = 2 * 12; // tail history delay at 48k = SINC_HALF_TAPS input samples… asserted below by best fit
+        let maxErr = 0;
+        for (let i = 500; i < out.length - 500; i++) {
+            const expected = Math.sin((2 * Math.PI * 1000 * (i - delaySamples * 1.5)) / 48000);
+            maxErr = Math.max(maxErr, Math.abs(out[i]! - expected));
+        }
+        expect(maxErr).toBeLessThan(0.01);
     });
 
     it('close() flushes and closes the context; empty chunks are ignored', () => {

src/voice-session.ts

@@ -105,6 +105,7 @@ export interface VoiceSessionSeams {
 export interface PlayerAudioContext {
     readonly currentTime: number;
     readonly destination: unknown;
+    readonly sampleRate: number;
     createBuffer(channels: number, length: number, sampleRate: number): { getChannelData(channel: number): Float32Array };
     createBufferSource(): {
         buffer: unknown;
@@ -116,29 +117,119 @@ export interface PlayerAudioContext {
     close?(): Promise<void>;
 }
 
+/** Half-width of the {@link StreamUpsampler} sinc kernel, in input samples. */
+export const SINC_HALF_TAPS = 12;
+
+/** Hann-windowed normalized sinc over ±{@link SINC_HALF_TAPS} input samples. */
+function sincWin(v: number): number {
+    if (v === 0) return 1;
+    const pv = Math.PI * v;
+    return (Math.sin(pv) / pv) * (0.5 + 0.5 * Math.cos(pv / SINC_HALF_TAPS));
+}
+
+/**
+ * Streaming band-limited upsampler `inRate` → `outRate` (windowed-sinc
+ * interpolation). Filter history and fractional phase carry across chunks, so
+ * a stream split at arbitrary chunk boundaries resamples identically to the
+ * unsplit stream — no boundary clicks.
+ *
+ * Exists because playing 16 kHz buffers through a forced-16 kHz AudioContext
+ * leaves the 16k→hardware-rate conversion to the browser's output resampler,
+ * and cheap interpolation there mirrors speech energy above 8 kHz — audible as
+ * harsh static riding the agent's voice (SMOODEV-2668).
+ */
+export class StreamUpsampler {
+    /** Input samples advanced per output sample (< 1 when upsampling). */
+    private readonly step: number;
+    private tail = new Float32Array(2 * SINC_HALF_TAPS);
+    private phase = 0;
+
+    constructor(inRate: number, outRate: number) {
+        this.step = inRate / outRate;
+    }
+
+    /** Resample one chunk; returns output samples ready to schedule. */
+    process(input: Float32Array): Float32Array {
+        const x = new Float32Array(this.tail.length + input.length);
+        x.set(this.tail);
+        x.set(input, this.tail.length);
+        const maxT = x.length - SINC_HALF_TAPS;
+        const outLen = Math.max(0, Math.ceil((maxT - SINC_HALF_TAPS - this.phase) / this.step));
+        const out = new Float32Array(outLen);
+        let t = this.phase + SINC_HALF_TAPS;
+        for (let o = 0; o < outLen; o++) {
+            const i0 = Math.floor(t);
+            const frac = t - i0;
+            let acc = 0;
+            // ponytail: per-sample sinc eval (~24 sin() per output sample);
+            // precompute a phase table if profiling ever cares.
+            for (let k = 1 - SINC_HALF_TAPS; k <= SINC_HALF_TAPS; k++) {
+                acc += x[i0 + k]! * sincWin(k - frac);
+            }
+            out[o] = acc;
+            t += this.step;
+        }
+        this.phase = t - maxT;
+        this.tail = x.slice(x.length - 2 * SINC_HALF_TAPS);
+        return out;
+    }
+}
+
+/**
+ * Seconds of audio pre-buffered when scheduling against a drained queue. A
+ * chunk that arrives just-in-time otherwise starts exactly at the playhead and
+ * the next network hiccup opens an audible gap (click).
+ */
+export const PLAYBACK_PRIME_S = 0.1;
+
 /**
  * Gapless PCM playback: each incoming linear16 chunk becomes an AudioBuffer
  * scheduled back-to-back (`nextTime`) so consecutive chunks butt together with
  * no gaps. `flush()` stops every scheduled source (barge-in / mic-button stop).
+ *
+ * The wire is one continuous 16 kHz linear16 byte stream — frames may split
+ * mid-sample, so a dangling byte is carried into the next frame (an odd-length
+ * `Int16Array` view would throw and every later frame would decode one byte
+ * off: pure noise). Chunks are resampled to the context's native rate via
+ * {@link StreamUpsampler} rather than trusting the browser's output resampler.
  */
 export class PcmPlayer implements VoicePlayer {
     private nextTime = 0;
+    private pendingByte: number | null = null;
+    private resampler: StreamUpsampler | null;
     private readonly live = new Set<ReturnType<PlayerAudioContext['createBufferSource']>>();
 
-    constructor(private readonly ctx: PlayerAudioContext) {}
+    constructor(private readonly ctx: PlayerAudioContext) {
+        this.resampler = ctx.sampleRate !== VOICE_SAMPLE_RATE ? new StreamUpsampler(VOICE_SAMPLE_RATE, ctx.sampleRate) : null;
+    }
 
     enqueue(chunk: ArrayBuffer): void {
-        const int16 = new Int16Array(chunk);
-        if (int16.length === 0) return;
-        const buf = this.ctx.createBuffer(1, int16.length, VOICE_SAMPLE_RATE);
-        const ch = buf.getChannelData(0);
-        for (let i = 0; i < int16.length; i++) ch[i] = int16[i]! / 32768;
+        // Re-align to sample boundaries across frames.
+        let bytes = new Uint8Array(chunk);
+        if (this.pendingByte !== null) {
+            const joined = new Uint8Array(bytes.length + 1);
+            joined[0] = this.pendingByte;
+            joined.set(bytes, 1);
+            this.pendingByte = null;
+            bytes = joined;
+        }
+        const evenLen = bytes.length & ~1;
+        if (evenLen < bytes.length) this.pendingByte = bytes[bytes.length - 1]!;
+        if (evenLen === 0) return;
+        const int16 = new Int16Array(bytes.buffer, bytes.byteOffset, evenLen / 2);
+        let samples: Float32Array = new Float32Array(int16.length);
+        for (let i = 0; i < int16.length; i++) samples[i] = int16[i]! / 32768;
+        if (this.resampler) samples = this.resampler.process(samples);
+        if (samples.length === 0) return;
+        const buf = this.ctx.createBuffer(1, samples.length, this.ctx.sampleRate);
+        buf.getChannelData(0).set(samples);
         const src = this.ctx.createBufferSource();
         src.buffer = buf;
         src.connect(this.ctx.destination);
-        const start = Math.max(this.ctx.currentTime, this.nextTime);
+        const now = this.ctx.currentTime;
+        const start = this.nextTime > now ? this.nextTime : now + PLAYBACK_PRIME_S;
         src.start(start);
-        this.nextTime = start + int16.length / VOICE_SAMPLE_RATE;
+        this.nextTime = start + samples.length / this.ctx.sampleRate;
         this.live.add(src);
         src.onended = () => this.live.delete(src);
     }
@@ -153,6 +244,8 @@ export class PcmPlayer implements VoicePlayer {
         }
         this.live.clear();
         this.nextTime = 0;
+        this.pendingByte = null;
+        if (this.resampler) this.resampler = new StreamUpsampler(VOICE_SAMPLE_RATE, this.ctx.sampleRate);
     }
 
     close(): void {
@@ -219,7 +312,15 @@ const defaultStartCapture: StartCapture = async (onFrame) => {
 const defaultCreatePlayer = (): VoicePlayer => {
     const Ctx = (globalThis as { AudioContext?: new (opts?: { sampleRate?: number }) => AudioContext }).AudioContext;
     if (!Ctx) throw new Error('AudioContext is not available');
-    return new PcmPlayer(new Ctx({ sampleRate: VOICE_SAMPLE_RATE }) as unknown as PlayerAudioContext);
+    // Native device rate — PcmPlayer upsamples 16 kHz → hardware itself
+    // (SMOODEV-2668). A sub-16 kHz device rate (exotic) would need a
+    // band-limiting DOWNsampler instead; force a 16 kHz context there.
+    let ctx = new Ctx();
+    if (ctx.sampleRate < VOICE_SAMPLE_RATE) {
+        void ctx.close();
+        ctx = new Ctx({ sampleRate: VOICE_SAMPLE_RATE });
+    }
+    return new PcmPlayer(ctx as unknown as PlayerAudioContext);
 };
 
 // ────────────────────────────── VoiceSession ────────────────────────────────