diff --git a/.changeset/voice-static-fix.md b/.changeset/voice-static-fix.md new file mode 100644 index 0000000..49da973 --- /dev/null +++ b/.changeset/voice-static-fix.md @@ -0,0 +1,5 @@ +--- +'@smooai/chat-widget': patch +--- + +Fix heavy static on browser-voice TTS playback (SMOODEV-2668): re-align linear16 frames that split mid-sample (an odd-length frame made every later frame decode one byte off — pure noise), resample 16 kHz audio to the AudioContext's native rate with a proper band-limited streaming upsampler instead of forcing a 16 kHz context (whose browser-side output resampling mirrored speech energy above 8 kHz as harsh imaging static), and prime playback ~100 ms off the playhead when the queue drains so just-in-time chunks don't click. diff --git a/src/voice-session.test.ts b/src/voice-session.test.ts index 287c762..25ad831 100644 --- a/src/voice-session.test.ts +++ b/src/voice-session.test.ts @@ -16,8 +16,10 @@ import { describe, expect, it } from 'vitest'; import { downsampleTo16k, PcmPlayer, + PLAYBACK_PRIME_S, type PlayerAudioContext, rmsLevel, + StreamUpsampler, VOICE_SAMPLE_RATE, type VoicePlayer, VoiceSession, @@ -364,17 +366,29 @@ interface ScheduledSource { length: number; } -/** Fake 16 kHz AudioContext capturing scheduling for assertions. */ -function makeFakeCtx(): { ctx: PlayerAudioContext; scheduled: ScheduledSource[]; setTime: (t: number) => void; closed: () => boolean } { +/** Fake AudioContext capturing scheduling + written samples for assertions. */ +function makeFakeCtx(sampleRate = 16000): { + ctx: PlayerAudioContext; + scheduled: ScheduledSource[]; + buffers: Float32Array[]; + setTime: (t: number) => void; + closed: () => boolean; +} { let now = 0; let closed = false; const scheduled: ScheduledSource[] = []; + const buffers: Float32Array[] = []; const ctx: PlayerAudioContext = { get currentTime() { return now; }, destination: {}, - createBuffer: (_ch, length) => ({ getChannelData: () => new Float32Array(length) }), + sampleRate, + createBuffer: (_ch, length) => { + const data = new Float32Array(length); + buffers.push(data); + return { getChannelData: () => data }; + }, createBufferSource: () => { const rec: ScheduledSource = { startedAt: undefined, stopped: false, length: 0 }; scheduled.push(rec); @@ -395,31 +409,31 @@ function makeFakeCtx(): { ctx: PlayerAudioContext; scheduled: ScheduledSource[]; return Promise.resolve(); }, }; - return { ctx, scheduled, setTime: (t) => (now = t), closed: () => closed }; + return { ctx, scheduled, buffers, setTime: (t) => (now = t), closed: () => closed }; } describe('PcmPlayer', () => { const chunk = (samples: number): ArrayBuffer => new Int16Array(samples).buffer as ArrayBuffer; - it('schedules consecutive chunks back-to-back (gapless)', () => { + it('schedules consecutive chunks back-to-back (gapless), primed off the playhead', () => { const { ctx, scheduled } = makeFakeCtx(); const player = new PcmPlayer(ctx); player.enqueue(chunk(1600)); // 100 ms player.enqueue(chunk(800)); // 50 ms player.enqueue(chunk(1600)); const starts = scheduled.map((s) => s.startedAt!); - expect(starts[0]).toBe(0); - expect(starts[1]).toBeCloseTo(0.1, 10); - expect(starts[2]).toBeCloseTo(0.15, 10); + expect(starts[0]).toBe(PLAYBACK_PRIME_S); + expect(starts[1]).toBeCloseTo(PLAYBACK_PRIME_S + 0.1, 10); + expect(starts[2]).toBeCloseTo(PLAYBACK_PRIME_S + 0.15, 10); }); - it('re-anchors to currentTime after the queue drains (no scheduling in the past)', () => { + it('re-anchors (primed) after the queue drains — no scheduling in the past', () => { const { ctx, scheduled, setTime } = makeFakeCtx(); const player = new PcmPlayer(ctx); - player.enqueue(chunk(1600)); // plays 0 → 0.1 + player.enqueue(chunk(1600)); // plays PRIME → PRIME+0.1 setTime(0.5); // long silence; next reply arrives later player.enqueue(chunk(1600)); - expect(scheduled[1]!.startedAt).toBe(0.5); + expect(scheduled[1]!.startedAt).toBeCloseTo(0.5 + PLAYBACK_PRIME_S, 10); }); it('flush() stops every live source and resets the schedule', () => { @@ -432,7 +446,65 @@ describe('PcmPlayer', () => { // After a flush the next chunk anchors to now, not the stale nextTime. setTime(0.05); player.enqueue(chunk(1600)); - expect(scheduled[2]!.startedAt).toBe(0.05); + expect(scheduled[2]!.startedAt).toBeCloseTo(0.05 + PLAYBACK_PRIME_S, 10); + }); + + it('reassembles samples split mid-sample across frames (odd-length chunks)', () => { + // The wire is one continuous byte stream; a frame boundary can land in + // the middle of an int16. Split [100, -200, 300, 400] as 3+5 bytes. + const { ctx, buffers } = makeFakeCtx(); + const player = new PcmPlayer(ctx); + const stream = new Uint8Array(new Int16Array([100, -200, 300, 400]).buffer); + player.enqueue(stream.slice(0, 3).buffer as ArrayBuffer); + player.enqueue(stream.slice(3).buffer as ArrayBuffer); + const played = buffers.flatMap((b) => Array.from(b)); + expect(played).toEqual([100 / 32768, -200 / 32768, 300 / 32768, 400 / 32768]); + }); + + it('upsamples to a non-16k context rate and advances the schedule in output time', () => { + const { ctx, scheduled, buffers } = makeFakeCtx(48000); + const player = new PcmPlayer(ctx); + player.enqueue(chunk(1600)); // 100 ms of input + const outLen = buffers[0]!.length; + // ~3× the input samples (minus the sinc kernel's startup history). + expect(outLen).toBeGreaterThan(4600); + expect(outLen).toBeLessThanOrEqual(4800); + expect(scheduled[0]!.startedAt).toBe(PLAYBACK_PRIME_S); + // nextTime advanced by outLen/48000: the follow-up chunk butts against it. + player.enqueue(chunk(1600)); + expect(scheduled[1]!.startedAt).toBeCloseTo(PLAYBACK_PRIME_S + outLen / 48000, 10); + }); + + it('upsampler output is identical whether the stream is split or not (no boundary artifacts)', () => { + // 440 Hz tone at 16 kHz, processed whole vs. split at awkward points — + // the carried filter state must make the outputs bit-identical. + const input = new Float32Array(1600); + for (let i = 0; i < input.length; i++) input[i] = Math.sin((2 * Math.PI * 440 * i) / VOICE_SAMPLE_RATE); + const whole = new StreamUpsampler(VOICE_SAMPLE_RATE, 48000).process(input); + const split = new StreamUpsampler(VOICE_SAMPLE_RATE, 48000); + const parts = [input.slice(0, 7), input.slice(7, 700), input.slice(700, 701), input.slice(701)].map((p) => split.process(p)); + const joined = parts.flatMap((p) => Array.from(p)); + expect(joined.length).toBe(whole.length); + for (let i = 0; i < whole.length; i++) { + expect(Math.abs(joined[i]! - whole[i]!)).toBeLessThan(1e-9); + } + }); + + it('upsampler reproduces an in-band tone faithfully (no imaging energy)', () => { + // A 1 kHz tone upsampled 16k → 48k should still be a clean 1 kHz tone: + // compare against the analytically expected samples (skip the kernel + // warm-up at the head). + const input = new Float32Array(3200); + for (let i = 0; i < input.length; i++) input[i] = Math.sin((2 * Math.PI * 1000 * i) / VOICE_SAMPLE_RATE); + const up = new StreamUpsampler(VOICE_SAMPLE_RATE, 48000); + const out = up.process(input); + const delaySamples = 2 * 12; // tail history delay at 48k = SINC_HALF_TAPS input samples… asserted below by best fit + let maxErr = 0; + for (let i = 500; i < out.length - 500; i++) { + const expected = Math.sin((2 * Math.PI * 1000 * (i - delaySamples * 1.5)) / 48000); + maxErr = Math.max(maxErr, Math.abs(out[i]! - expected)); + } + expect(maxErr).toBeLessThan(0.01); }); it('close() flushes and closes the context; empty chunks are ignored', () => { diff --git a/src/voice-session.ts b/src/voice-session.ts index d0b4bc2..5de85de 100644 --- a/src/voice-session.ts +++ b/src/voice-session.ts @@ -105,6 +105,7 @@ export interface VoiceSessionSeams { export interface PlayerAudioContext { readonly currentTime: number; readonly destination: unknown; + readonly sampleRate: number; createBuffer(channels: number, length: number, sampleRate: number): { getChannelData(channel: number): Float32Array }; createBufferSource(): { buffer: unknown; @@ -116,29 +117,119 @@ export interface PlayerAudioContext { close?(): Promise; } +/** Half-width of the {@link StreamUpsampler} sinc kernel, in input samples. */ +export const SINC_HALF_TAPS = 12; + +/** Hann-windowed normalized sinc over ±{@link SINC_HALF_TAPS} input samples. */ +function sincWin(v: number): number { + if (v === 0) return 1; + const pv = Math.PI * v; + return (Math.sin(pv) / pv) * (0.5 + 0.5 * Math.cos(pv / SINC_HALF_TAPS)); +} + +/** + * Streaming band-limited upsampler `inRate` → `outRate` (windowed-sinc + * interpolation). Filter history and fractional phase carry across chunks, so + * a stream split at arbitrary chunk boundaries resamples identically to the + * unsplit stream — no boundary clicks. + * + * Exists because playing 16 kHz buffers through a forced-16 kHz AudioContext + * leaves the 16k→hardware-rate conversion to the browser's output resampler, + * and cheap interpolation there mirrors speech energy above 8 kHz — audible as + * harsh static riding the agent's voice (SMOODEV-2668). + */ +export class StreamUpsampler { + /** Input samples advanced per output sample (< 1 when upsampling). */ + private readonly step: number; + private tail = new Float32Array(2 * SINC_HALF_TAPS); + private phase = 0; + + constructor(inRate: number, outRate: number) { + this.step = inRate / outRate; + } + + /** Resample one chunk; returns output samples ready to schedule. */ + process(input: Float32Array): Float32Array { + const x = new Float32Array(this.tail.length + input.length); + x.set(this.tail); + x.set(input, this.tail.length); + const maxT = x.length - SINC_HALF_TAPS; + const outLen = Math.max(0, Math.ceil((maxT - SINC_HALF_TAPS - this.phase) / this.step)); + const out = new Float32Array(outLen); + let t = this.phase + SINC_HALF_TAPS; + for (let o = 0; o < outLen; o++) { + const i0 = Math.floor(t); + const frac = t - i0; + let acc = 0; + // ponytail: per-sample sinc eval (~24 sin() per output sample); + // precompute a phase table if profiling ever cares. + for (let k = 1 - SINC_HALF_TAPS; k <= SINC_HALF_TAPS; k++) { + acc += x[i0 + k]! * sincWin(k - frac); + } + out[o] = acc; + t += this.step; + } + this.phase = t - maxT; + this.tail = x.slice(x.length - 2 * SINC_HALF_TAPS); + return out; + } +} + +/** + * Seconds of audio pre-buffered when scheduling against a drained queue. A + * chunk that arrives just-in-time otherwise starts exactly at the playhead and + * the next network hiccup opens an audible gap (click). + */ +export const PLAYBACK_PRIME_S = 0.1; + /** * Gapless PCM playback: each incoming linear16 chunk becomes an AudioBuffer * scheduled back-to-back (`nextTime`) so consecutive chunks butt together with * no gaps. `flush()` stops every scheduled source (barge-in / mic-button stop). + * + * The wire is one continuous 16 kHz linear16 byte stream — frames may split + * mid-sample, so a dangling byte is carried into the next frame (an odd-length + * `Int16Array` view would throw and every later frame would decode one byte + * off: pure noise). Chunks are resampled to the context's native rate via + * {@link StreamUpsampler} rather than trusting the browser's output resampler. */ export class PcmPlayer implements VoicePlayer { private nextTime = 0; + private pendingByte: number | null = null; + private resampler: StreamUpsampler | null; private readonly live = new Set>(); - constructor(private readonly ctx: PlayerAudioContext) {} + constructor(private readonly ctx: PlayerAudioContext) { + this.resampler = ctx.sampleRate !== VOICE_SAMPLE_RATE ? new StreamUpsampler(VOICE_SAMPLE_RATE, ctx.sampleRate) : null; + } enqueue(chunk: ArrayBuffer): void { - const int16 = new Int16Array(chunk); - if (int16.length === 0) return; - const buf = this.ctx.createBuffer(1, int16.length, VOICE_SAMPLE_RATE); - const ch = buf.getChannelData(0); - for (let i = 0; i < int16.length; i++) ch[i] = int16[i]! / 32768; + // Re-align to sample boundaries across frames. + let bytes = new Uint8Array(chunk); + if (this.pendingByte !== null) { + const joined = new Uint8Array(bytes.length + 1); + joined[0] = this.pendingByte; + joined.set(bytes, 1); + this.pendingByte = null; + bytes = joined; + } + const evenLen = bytes.length & ~1; + if (evenLen < bytes.length) this.pendingByte = bytes[bytes.length - 1]!; + if (evenLen === 0) return; + const int16 = new Int16Array(bytes.buffer, bytes.byteOffset, evenLen / 2); + let samples: Float32Array = new Float32Array(int16.length); + for (let i = 0; i < int16.length; i++) samples[i] = int16[i]! / 32768; + if (this.resampler) samples = this.resampler.process(samples); + if (samples.length === 0) return; + const buf = this.ctx.createBuffer(1, samples.length, this.ctx.sampleRate); + buf.getChannelData(0).set(samples); const src = this.ctx.createBufferSource(); src.buffer = buf; src.connect(this.ctx.destination); - const start = Math.max(this.ctx.currentTime, this.nextTime); + const now = this.ctx.currentTime; + const start = this.nextTime > now ? this.nextTime : now + PLAYBACK_PRIME_S; src.start(start); - this.nextTime = start + int16.length / VOICE_SAMPLE_RATE; + this.nextTime = start + samples.length / this.ctx.sampleRate; this.live.add(src); src.onended = () => this.live.delete(src); } @@ -153,6 +244,8 @@ export class PcmPlayer implements VoicePlayer { } this.live.clear(); this.nextTime = 0; + this.pendingByte = null; + if (this.resampler) this.resampler = new StreamUpsampler(VOICE_SAMPLE_RATE, this.ctx.sampleRate); } close(): void { @@ -219,7 +312,15 @@ const defaultStartCapture: StartCapture = async (onFrame) => { const defaultCreatePlayer = (): VoicePlayer => { const Ctx = (globalThis as { AudioContext?: new (opts?: { sampleRate?: number }) => AudioContext }).AudioContext; if (!Ctx) throw new Error('AudioContext is not available'); - return new PcmPlayer(new Ctx({ sampleRate: VOICE_SAMPLE_RATE }) as unknown as PlayerAudioContext); + // Native device rate — PcmPlayer upsamples 16 kHz → hardware itself + // (SMOODEV-2668). A sub-16 kHz device rate (exotic) would need a + // band-limiting DOWNsampler instead; force a 16 kHz context there. + let ctx = new Ctx(); + if (ctx.sampleRate < VOICE_SAMPLE_RATE) { + void ctx.close(); + ctx = new Ctx({ sampleRate: VOICE_SAMPLE_RATE }); + } + return new PcmPlayer(ctx as unknown as PlayerAudioContext); }; // ────────────────────────────── VoiceSession ────────────────────────────────