Skip to content

Commit f1db4dd

Browse files
zcoderrclaude
andcommitted
fix: add VAD config and audio relay logging for voice chat
- Add realtimeInputConfig with high sensitivity VAD to Gemini setup - Log audio chunk flow and non-serverContent Gemini messages for debugging Co-authored-by: Claude <noreply@anthropic.com>
1 parent 214dfe8 commit f1db4dd

1 file changed

Lines changed: 27 additions & 1 deletion

File tree

api/src/routes/voice/relay.ts

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,17 +113,36 @@ export class GeminiLiveRelay {
113113
systemInstruction: {
114114
parts: [{ text: this.session.systemInstruction }],
115115
},
116+
realtimeInputConfig: {
117+
automaticActivityDetection: {
118+
disabled: false,
119+
startOfSpeechSensitivity: 'START_SENSITIVITY_HIGH',
120+
endOfSpeechSensitivity: 'END_SENSITIVITY_HIGH',
121+
prefixPaddingMs: 20,
122+
silenceDurationMs: 500,
123+
},
124+
},
116125
},
117126
};
118127
this.geminiWs?.send(JSON.stringify(setup));
119128
}
120129

130+
private clientChunkCount = 0;
131+
121132
/** Handle messages from the client */
122133
private handleClientMessage(raw: WebSocket.Data): void {
123134
try {
124135
const msg = JSON.parse(raw.toString());
125136

126137
if (msg.type === 'audio' && this.isSetupComplete) {
138+
this.clientChunkCount++;
139+
if (this.clientChunkCount === 1 || this.clientChunkCount % 100 === 0) {
140+
logger.info('[voice-relay] audio chunk', {
141+
n: this.clientChunkCount,
142+
bytes: msg.data?.length ?? 0,
143+
userId: this.session.userId,
144+
});
145+
}
127146
// Forward audio to Gemini in realtimeInput format
128147
this.geminiWs?.send(
129148
JSON.stringify({
@@ -157,7 +176,14 @@ export class GeminiLiveRelay {
157176
}
158177

159178
const sc = msg.serverContent;
160-
if (!sc) return;
179+
if (!sc) {
180+
// Log unexpected Gemini messages (errors, tool calls, etc.)
181+
const keys = Object.keys(msg).join(',');
182+
if (keys !== 'setupComplete') {
183+
logger.info('[voice-relay] gemini non-serverContent msg', { keys });
184+
}
185+
return;
186+
}
161187

162188
// Forward audio data
163189
if (sc.modelTurn?.parts) {

0 commit comments

Comments
 (0)