From 5a449f84518a31c20fdb1481b4b9401e333322d7 Mon Sep 17 00:00:00 2001 From: Brent Rager Date: Sat, 11 Jul 2026 17:33:37 -0400 Subject: [PATCH] =?UTF-8?q?SMOODEV-2534:=20voice=20input/output=20?= =?UTF-8?q?=E2=80=94=20mic=20capture=20+=20TTS=20playback=20over=20the=20b?= =?UTF-8?q?rowser-voice=20WS=20(flag=20OFF)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a framework-free VoiceSession speaking the frozen browser-voice protocol (start/interrupt/stop JSON + 16 kHz linear16 binary both ways), a composer mic toggle gated by the new voice.enabled config (OFF by default, zero UI when off), RMS barge-in that interrupts + flushes playback, and voice turns rendered through the existing message path so the transcript stays coherent with text chat. Co-Authored-By: Claude Fable 5 --- .changeset/voice-input-output.md | 5 + src/config.ts | 19 +- src/conversation.ts | 24 +- src/element.test.ts | 77 ++++++ src/element.ts | 115 +++++++- src/index.ts | 20 +- src/styles.ts | 41 +++ src/voice-session.test.ts | 448 +++++++++++++++++++++++++++++++ src/voice-session.ts | 420 +++++++++++++++++++++++++++++ 9 files changed, 1165 insertions(+), 4 deletions(-) create mode 100644 .changeset/voice-input-output.md create mode 100644 src/voice-session.test.ts create mode 100644 src/voice-session.ts diff --git a/.changeset/voice-input-output.md b/.changeset/voice-input-output.md new file mode 100644 index 0000000..d709852 --- /dev/null +++ b/.changeset/voice-input-output.md @@ -0,0 +1,5 @@ +--- +'@smooai/chat-widget': minor +--- + +Voice input/output (SMOODEV-2534): mic capture + TTS playback over the browser-voice WebSocket, shipped dark behind the new `voice: { enabled, url? }` config option (OFF by default — zero UI when off). Adds `VoiceSession` (framework-free WS lifecycle, getUserMedia → AudioWorklet/ScriptProcessor → 16 kHz linear16 mic streaming, gapless PCM playback, RMS barge-in with `interrupt` + playback flush), an Aurora Glass mic toggle in the composer with listening/speaking indicators and a live partial transcript in the input, and voice turns (`transcript_final` / `reply_text`) rendered through the normal chat message path. The text session's conversation id is passed as `conversation_id` so voice resumes the same thread. diff --git a/src/config.ts b/src/config.ts index 39ce2a8..77c4575 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,6 +6,19 @@ * {@link mountChatWidget} / `element.configure(...)`). */ import { safeHttpUrl } from './markdown.js'; +import { DEFAULT_VOICE_URL } from './voice-session.js'; + +/** + * Browser voice input/output (SMOODEV-2534). OFF by default — when disabled the + * widget renders zero voice UI. When enabled, a mic toggle appears in the + * composer and speech flows over the browser-voice WebSocket. + */ +export interface ChatWidgetVoiceConfig { + /** Turn the voice feature on. Default `false`. */ + enabled?: boolean; + /** Browser-voice WS endpoint. Defaults to the hosted SmooAI voice service. */ + url?: string; +} export interface ChatWidgetTheme { /** Foreground text color for the widget chrome. */ @@ -151,6 +164,8 @@ export interface ChatWidgetConfig { * surfaces where seeing what the agent did mid-turn is valuable. */ showToolActivity?: boolean; + /** Browser voice input/output. OFF by default (zero UI when off). */ + voice?: ChatWidgetVoiceConfig; /** Theme overrides. */ theme?: ChatWidgetTheme; } @@ -158,8 +173,9 @@ export interface ChatWidgetConfig { /** The fully-resolved theme (canonical keys only — aliases are folded in). */ export type ResolvedTheme = Required>; -export type ResolvedConfig = Required> & { +export type ResolvedConfig = Required> & { theme: ResolvedTheme; + voice: { enabled: boolean; url: string }; userName?: string; userEmail?: string; userPhone?: string; @@ -205,6 +221,7 @@ export function resolveConfig(config: ChatWidgetConfig): ResolvedConfig { allowChatRestore: config.allowChatRestore ?? true, allowAnonymous: config.allowAnonymous ?? false, showToolActivity: config.showToolActivity ?? false, + voice: { enabled: config.voice?.enabled ?? false, url: config.voice?.url ?? DEFAULT_VOICE_URL }, theme: { text: theme.text ?? '#f8fafc', background: theme.background ?? '#040d30', diff --git a/src/conversation.ts b/src/conversation.ts index 30ba300..724f393 100644 --- a/src/conversation.ts +++ b/src/conversation.ts @@ -328,6 +328,8 @@ export class ConversationController { private readonly store: StoreApi; private client: SmoothAgentClient | null = null; private sessionId: string | null = null; + /** Conversation id of the live session (create or resume) — lets voice join the same thread. */ + private conversationId: string | null = null; private readonly messages: ChatMessage[] = []; private status: ConnectionStatus = 'idle'; private seq = 0; @@ -384,6 +386,23 @@ export class ConversationController { return this.status; } + /** Conversation id of the live session, or null before connect (voice passes this as `conversation_id`). */ + get currentConversationId(): string | null { + return this.conversationId; + } + + /** + * Append an already-finalized message to the transcript and emit — the voice + * path reuses this so `transcript_final` (user) and `reply_text` (assistant) + * turns land in the same message list / render pipeline as typed chat. + */ + appendLocalMessage(role: Role, text: string): void { + const trimmed = text.trim(); + if (!trimmed) return; + this.messages.push({ id: this.nextId(role === 'user' ? 'u' : 'a'), role, text: trimmed, streaming: false }); + this.emitMessages(); + } + /** The persisted store, exposed so the view can read identity for the pre-chat gate. */ getStore(): StoreApi { return this.store; @@ -690,6 +709,7 @@ export class ConversationController { ...(metadata ? { metadata } : {}), }); this.sessionId = session.sessionId; + this.conversationId = session.conversationId ?? null; this.store.getState().setSessionId(session.sessionId); } @@ -699,7 +719,7 @@ export class ConversationController { */ private async tryResume(sessionId: string): Promise { if (!this.client) return false; - let snap: { status?: 'active' | 'idle' | 'ended' }; + let snap: { status?: 'active' | 'idle' | 'ended'; conversationId?: string }; try { snap = await this.client.getSession({ sessionId }); } catch { @@ -708,6 +728,7 @@ export class ConversationController { if (snap.status === 'ended') return false; this.sessionId = sessionId; + this.conversationId = snap.conversationId ?? null; await this.hydrateHistory(sessionId); return true; } @@ -1052,6 +1073,7 @@ export class ConversationController { this.client?.disconnect('widget closed'); this.client = null; this.sessionId = null; + this.conversationId = null; this.activeRequestId = null; // A full teardown ends the controller lifecycle: a subsequent connect() is a // genuine re-open and may resume again, so re-arm the resume probe. diff --git a/src/element.test.ts b/src/element.test.ts index 97f2fb4..24c4cfa 100644 --- a/src/element.test.ts +++ b/src/element.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import { defineChatWidget, ELEMENT_TAG, INTERACTION_CARDS, safeHttpUrl } from './element.js'; import { type Interrupt, SUPPORTED_INTERACTION_CAPABILITIES } from './conversation.js'; +import { VoiceSession } from './voice-session.js'; describe('safeHttpUrl', () => { it('accepts absolute http(s) URLs and returns the normalized href', () => { @@ -470,6 +471,82 @@ describe('fullpage container sizing (composer-clip regression)', () => { }); }); +describe('voice config gating (SMOODEV-2534)', () => { + afterEach(() => { + document.body.innerHTML = ''; + }); + + function mountCfg(cfg: Record): HTMLElement { + defineChatWidget(); + const el = document.createElement(ELEMENT_TAG) as HTMLElement & { configure: (c: Record) => void }; + el.setAttribute('endpoint', 'wss://e/ws'); + el.setAttribute('agent-id', 'a1'); + el.configure(cfg); + document.body.appendChild(el); + return el; + } + + it('renders NO mic button by default (voice OFF = zero voice UI)', () => { + const sr = mountCfg({}).shadowRoot!; + expect(sr.querySelector('.composer .send')).not.toBeNull(); + expect(sr.querySelector('.composer .mic')).toBeNull(); + }); + + it('renders no mic button when voice is explicitly disabled', () => { + const sr = mountCfg({ voice: { enabled: false } }).shadowRoot!; + expect(sr.querySelector('.composer .mic')).toBeNull(); + }); + + it('renders the mic toggle in the composer when voice is enabled', () => { + const sr = mountCfg({ voice: { enabled: true } }).shadowRoot!; + const mic = sr.querySelector('.composer .mic') as HTMLButtonElement | null; + expect(mic).not.toBeNull(); + expect(mic?.getAttribute('aria-pressed')).toBe('false'); + expect(mic?.getAttribute('aria-label')).toBe('Start voice'); + expect(mic?.querySelector('svg')).not.toBeNull(); + }); + + it('starting voice passes the configured url + agentId and lights the button', async () => { + // No real audio/WS in jsdom — stub the session's start; the element's + // wiring (options threaded, UI state) is what's under test here. + const origStart = VoiceSession.prototype.start; + VoiceSession.prototype.start = () => Promise.resolve(); + try { + await runStartVoiceScenario(); + } finally { + VoiceSession.prototype.start = origStart; + } + }); + + async function runStartVoiceScenario(): Promise { + const el = mountCfg({ voice: { enabled: true, url: 'wss://voice.test/ws' } }); + const sr = el.shadowRoot!; + const priv = el as unknown as { + startVoice: () => Promise; + voiceSession: { stop: () => void } | null; + controller: unknown; + }; + // Stub the controller seam (same pattern as the other element tests). + priv.controller = { + currentConversationId: 'conv-42', + appendLocalMessage: () => {}, + connect: () => Promise.resolve(), + disconnect: () => {}, + }; + await priv.startVoice(); + const session = priv.voiceSession as unknown as { opts: { url?: string; agentId: string; conversationId?: string } } | null; + expect(session).not.toBeNull(); + expect(session!.opts.url).toBe('wss://voice.test/ws'); + expect(session!.opts.agentId).toBe('a1'); + // The text thread's conversation id rides along so voice resumes it. + expect(session!.opts.conversationId).toBe('conv-42'); + const mic = sr.querySelector('.composer .mic') as HTMLButtonElement; + expect(mic.classList.contains('active')).toBe(true); + expect(mic.getAttribute('aria-pressed')).toBe('true'); + expect(mic.getAttribute('aria-label')).toBe('Stop voice'); + } +}); + describe('Rich Interactions card registry', () => { afterEach(() => { document.body.innerHTML = ''; diff --git a/src/element.ts b/src/element.ts index efbfdf7..eddc709 100644 --- a/src/element.ts +++ b/src/element.ts @@ -30,6 +30,7 @@ import { import { SMOOTH_ICON_SVG } from './logo.js'; import { cleanCitationSnippet, escapeHtml, renderMarkdown, safeHttpUrl } from './markdown.js'; import { buildStyles } from './styles.js'; +import { VoiceSession } from './voice-session.js'; export const ELEMENT_TAG = 'smooth-agent-chat'; @@ -86,6 +87,8 @@ const ICON = { shield: ``, /** Tool-activity chip — a wrench. */ tool: ``, + /** Voice toggle — a microphone. */ + mic: ``, } as const; /** @@ -277,6 +280,10 @@ export class SmoothAgentChatElement extends HTMLElement { private allowChatRestore = true; /** True while the pre-chat identity gate is showing (blocks premature connect). */ private gating = false; + /** Voice config (SMOODEV-2534) — enabled=false renders zero voice UI. */ + private voiceCfg: { enabled: boolean; url: string } = { enabled: false, url: '' }; + /** Live voice session, or null when voice is off. */ + private voiceSession: VoiceSession | null = null; // Cached DOM refs (populated in render()). private panelEl: HTMLElement | null = null; @@ -286,6 +293,7 @@ export class SmoothAgentChatElement extends HTMLElement { private dotEl: HTMLElement | null = null; private inputEl: HTMLTextAreaElement | null = null; private sendBtn: HTMLButtonElement | null = null; + private micBtn: HTMLButtonElement | null = null; private suggestionsEl: HTMLElement | null = null; // ── Smooth streaming reveal ── @@ -321,6 +329,7 @@ export class SmoothAgentChatElement extends HTMLElement { disconnectedCallback(): void { this.mounted = false; this.resetReveal(); + this.stopVoice(); this.controller?.disconnect(); this.controller = null; } @@ -392,6 +401,7 @@ export class SmoothAgentChatElement extends HTMLElement { allowChatRestore: this.overrides.allowChatRestore, allowAnonymous: this.overrides.allowAnonymous, showToolActivity: this.overrides.showToolActivity ?? this.hasAttribute('show-tool-activity'), + voice: this.overrides.voice, theme, }; } @@ -527,6 +537,11 @@ export class SmoothAgentChatElement extends HTMLElement { const restoreBtn = this.allowChatRestore ? `` : ''; const footerInner = [brandingHtml, restoreBtn].filter(Boolean).join(' · '); const footerHtml = footerInner ? `
${footerInner}
` : ''; + // Voice (SMOODEV-2534): mic toggle in the composer, only when enabled. + this.voiceCfg = resolved.voice; + const micHtml = resolved.voice.enabled + ? `` + : ''; const chatHtml = `
@@ -534,6 +549,7 @@ export class SmoothAgentChatElement extends HTMLElement {
+ ${micHtml}
${footerHtml} @@ -555,8 +571,10 @@ export class SmoothAgentChatElement extends HTMLElement { if (logoSvg) logoSvg.setAttribute('class', 'logo'); // A full DOM rebuild invalidates any cached streaming-bubble ref; cancel - // the in-flight reveal loop so renderMessages can re-bind cleanly. + // the in-flight reveal loop so renderMessages can re-bind cleanly. A live + // voice session is bound to the old composer's mic button — end it too. this.resetReveal(); + this.stopVoice(); this.root.replaceChildren(style, container); this.launcherEl = container.querySelector('.launcher'); @@ -566,12 +584,14 @@ export class SmoothAgentChatElement extends HTMLElement { this.dotEl = container.querySelector('.dot'); this.inputEl = container.querySelector('textarea'); this.sendBtn = container.querySelector('.send'); + this.micBtn = container.querySelector('.mic'); this.interruptEl = container.querySelector('.interrupt'); this.suggestionsEl = container.querySelector('.reply-suggestions'); this.launcherEl?.addEventListener('click', () => this.openChat()); container.querySelector('.close')?.addEventListener('click', () => this.closeChat()); this.sendBtn?.addEventListener('click', () => this.submit()); + this.micBtn?.addEventListener('click', () => this.toggleVoice()); this.inputEl?.addEventListener('input', () => this.autosize()); this.inputEl?.addEventListener('keydown', (ev) => { if (ev.key === 'Enter' && !ev.shiftKey) { @@ -1525,6 +1545,99 @@ export class SmoothAgentChatElement extends HTMLElement { if (this.inputEl) this.inputEl.disabled = busy; } + // ───────────────────────── Voice (SMOODEV-2534) ───────────────────────────── + + /** + * Mic button: start a voice session, or — when one is live — end it. Hitting + * the button while the agent's TTS is playing barges in first (interrupt + + * playback flush) so the audio dies instantly, then the session ends. + */ + private toggleVoice(): void { + if (this.voiceSession) { + if (this.voiceSession.isSpeaking) this.voiceSession.interrupt(); + this.stopVoice(); + return; + } + void this.startVoice(); + } + + private async startVoice(): Promise { + if (!this.controller || this.voiceSession || !this.voiceCfg.enabled) return; + const config = this.readConfig(); + if (!config) return; + // Best-effort: join the text thread when one exists. If the text session + // hasn't connected yet, voice starts a fresh thread (the frozen protocol + // never returns the voice-created conversation id, so it can't be adopted + // for later text turns — tracked as a follow-up on the server protocol). + const conversationId = this.controller.currentConversationId ?? undefined; + const controller = this.controller; + const session = new VoiceSession( + { url: this.voiceCfg.url, agentId: config.agentId, conversationId }, + { + onTranscriptPartial: (text) => { + // Live partial transcript in the input area while listening. + if (this.inputEl) { + this.inputEl.value = text; + this.autosize(); + } + }, + onTranscriptFinal: (text) => { + if (this.inputEl) { + this.inputEl.value = ''; + this.autosize(); + } + // Finalized speech lands as a normal user message. + controller.appendLocalMessage('user', text); + }, + onReplyText: (text) => { + // Agent replies render through the normal chat message path. + controller.appendLocalMessage('assistant', text); + }, + onSpeaking: (speaking) => { + this.micBtn?.classList.toggle('speaking', speaking); + }, + onError: () => this.stopVoice(), + onEnded: () => { + // Server-side end (handoff / close) — reset the UI. stopVoice() + // is idempotent, so a local stop lands here harmlessly too. + this.voiceSession = null; + this.syncVoiceUi(false); + }, + }, + ); + this.voiceSession = session; + this.syncVoiceUi(true); + try { + await session.start(); + } catch { + // Mic permission denied / audio unavailable — back to text. + this.voiceSession = null; + this.syncVoiceUi(false); + } + } + + /** End any live voice session and reset the composer UI. Idempotent. */ + private stopVoice(): void { + const session = this.voiceSession; + this.voiceSession = null; + session?.stop(); + this.syncVoiceUi(false); + } + + /** Toggle the mic button's listening state + clear the partial transcript. */ + private syncVoiceUi(active: boolean): void { + this.micBtn?.classList.toggle('active', active); + this.micBtn?.setAttribute('aria-pressed', String(active)); + this.micBtn?.setAttribute('aria-label', active ? 'Stop voice' : 'Start voice'); + if (!active) { + this.micBtn?.classList.remove('speaking'); + if (this.inputEl && this.inputEl.value) { + this.inputEl.value = ''; + this.autosize(); + } + } + } + private submit(): void { if (!this.inputEl || !this.controller) return; const text = this.inputEl.value; diff --git a/src/index.ts b/src/index.ts index 206c111..7f4634a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,7 +26,25 @@ export { mountChatWidget, mountFullPageChat, } from './element.js'; -export type { ChatWidgetConfig, ChatWidgetMode, ChatWidgetTheme } from './config.js'; +export type { ChatWidgetConfig, ChatWidgetMode, ChatWidgetTheme, ChatWidgetVoiceConfig } from './config.js'; +// Browser voice input/output (SMOODEV-2534) — mic capture + TTS playback over +// the browser-voice WS. Gated by `voice.enabled` (OFF by default). +export { + DEFAULT_VOICE_URL, + downsampleTo16k, + PcmPlayer, + rmsLevel, + VOICE_SAMPLE_RATE, + VoiceSession, + type PlayerAudioContext, + type StartCapture, + type VoicePlayer, + type VoiceSessionEvents, + type VoiceSessionOptions, + type VoiceSessionSeams, + type VoiceSessionState, + type VoiceWebSocket, +} from './voice-session.js'; // The deferred loader (also shipped as the `./loader` IIFE for `