feat: add Gemini Live real-time voice chat via WebSocket relay

Add backend WebSocket relay that bridges iOS client to Gemini Live API
for real-time voice conversation, replacing the turn-based STT→Chat→TTS
voice flow with a single bidirectional audio stream.

Backend:
- POST /api/voice/session: creates session with JWT containing system
  instruction (Susan Miller voice persona + user astrology context)
- WebSocket /api/voice/stream: authenticates via JWT, relays PCM audio
  between client and Gemini Live API
- Voice-optimized prompt (no emoji, short conversational sentences)
- MessageType.VoiceChat = 25 for tracking voice sessions

Co-authored-by: Claude <noreply@anthropic.com>

Author: zcoderr

api/src/ai/voice-model.ts

@@ -0,0 +1,45 @@
+import Config from '@blocklet/sdk/lib/config';
+
+// ---------------------------------------------------------------------------
+// Gemini Live API configuration
+// Uses direct Google WebSocket API, not AIGNE Hub.
+// ---------------------------------------------------------------------------
+
+const DEFAULT_LIVE_MODEL = 'gemini-3.1-flash-live-preview';
+const DEFAULT_VOICE = 'Aoede';
+
+export function getGeminiApiKey(): string {
+  const key =
+    (Config.env.preferences?.gemini_api_key as string) ||
+    process.env.GEMINI_API_KEY ||
+    process.env.GOOGLE_API_KEY ||
+    '';
+  return key;
+}
+
+export function getGeminiLiveModel(): string {
+  return (Config.env.preferences?.gemini_live_model as string) || process.env.GEMINI_LIVE_MODEL || DEFAULT_LIVE_MODEL;
+}
+
+export function getGeminiLiveVoice(): string {
+  return (Config.env.preferences?.gemini_live_voice as string) || process.env.GEMINI_LIVE_VOICE || DEFAULT_VOICE;
+}
+
+export function getGeminiLiveWsUrl(): string {
+  const apiKey = getGeminiApiKey();
+  return (
+    'wss://generativelanguage.googleapis.com/ws/' +
+    'google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent' +
+    `?key=${apiKey}`
+  );
+}
+
+/** Maximum voice session duration in seconds (default 10 minutes) */
+export function getVoiceMaxDuration(): number {
+  return Number(Config.env.preferences?.voice_max_duration || process.env.VOICE_MAX_DURATION || 600);
+}
+
+/** Voice session token TTL in seconds (default 30 minutes) */
+export function getVoiceSessionTTL(): number {
+  return Number(Config.env.preferences?.voice_session_ttl || process.env.VOICE_SESSION_TTL || 1800);
+}

api/src/index.ts

@@ -6,6 +6,7 @@ import express, { ErrorRequestHandler } from 'express';
 import 'express-async-errors';
 import Joi from 'joi';
 import path from 'path';
+import { WebSocketServer } from 'ws';
 
 import { HttpError } from './libs/auth';
 import startBlogCron from './libs/cron-blog';
@@ -19,6 +20,7 @@ import aigneCompat from './routes/aigne-compat';
 // Mount daily routes at /daily/api to match the rewritten path
 import dailyRouter from './routes/daily';
 import routesV2 from './routes/v2';
+import { handleVoiceWebSocket } from './routes/voice/ws';
 
 dotenv.config();
 
@@ -79,6 +81,11 @@ export const server = app.listen(port, (err?: any) => {
   if (err) throw err;
   logger.info(`> ${name} v${version} ready on ${port}`);
 
+  // Mount WebSocket server for voice relay
+  const wss = new WebSocketServer({ server, path: '/api/voice/stream' });
+  wss.on('connection', handleVoiceWebSocket);
+  logger.info('> Voice WebSocket server mounted on /api/voice/stream');
+
   startBlogCron();
   startNotificationCron();
   startScribeCron();

api/src/routes/index.ts

@@ -15,12 +15,14 @@ import setting from './setting';
 import sns from './sns';
 import upload from './upload';
 import user from './user';
+import voice from './voice';
 
 const router = Router();
 
 router.use('/auth', auth);
 router.use('/user', user);
 router.use('/ai', ai);
+router.use('/voice', voice);
 router.use('/setting', setting);
 router.use('/upload', upload);
 router.use('/message', message);

api/src/routes/voice/index.ts

@@ -0,0 +1,71 @@
+import { user } from '@blocklet/sdk/lib/middlewares/user';
+import { Router } from 'express';
+import { sign } from 'jsonwebtoken';
+
+import { getGeminiApiKey, getGeminiLiveVoice, getVoiceSessionTTL } from '../../ai/voice-model';
+import { UnauthorizedError } from '../../libs/auth';
+import { config } from '../../libs/env';
+import { getLanguage } from '../../libs/language';
+import logger from '../../libs/logger';
+import Message, { MessageRole, MessageType } from '../../store/models/message';
+import { buildVoiceSystemInstruction } from './system-prompt';
+
+const router = Router();
+
+// JWT secret for voice session tokens — reuse the VOICE_SESSION_SECRET env var
+// or fall back to a derived key from the Gemini API key
+function getSessionSecret(): string {
+  return process.env.VOICE_SESSION_SECRET || `voice-session:${getGeminiApiKey()}`;
+}
+
+// ---------------------------------------------------------------------------
+// POST /api/voice/session — Create a voice session
+// Returns a signed JWT containing the system instruction for the WebSocket relay.
+// ---------------------------------------------------------------------------
+
+router.post('/session', user(), async (req, res) => {
+  const { did: userId } = req.user ?? {};
+  if (!userId) throw new UnauthorizedError();
+
+  const language = getLanguage(req);
+
+  // Check that Gemini API key is configured
+  const apiKey = getGeminiApiKey();
+  if (!apiKey) {
+    res.status(503).json({ error: { message: 'Voice feature is not configured' } });
+    return;
+  }
+
+  // Rate limit check — voice sessions share the same daily quota as SessionChat
+  const [remainingCount] = await Message.remainingQueryCount({ userId });
+  if (typeof remainingCount === 'number' && remainingCount <= 0) {
+    if (config.limitation.countMessageType.includes(MessageType.VoiceChat)) {
+      res.status(429).json({ error: { message: 'Daily query limit reached' } });
+      return;
+    }
+  }
+
+  // Build the voice system instruction with full user context
+  const systemInstruction = await buildVoiceSystemInstruction({ userId, language });
+  const voiceName = getGeminiLiveVoice();
+  const ttl = getVoiceSessionTTL();
+
+  // Create a message record for this voice session
+  await Message.bulkCreate([
+    { userId, type: MessageType.VoiceChat, role: MessageRole.user, content: '' },
+    { userId, type: MessageType.VoiceChat, role: MessageRole.ai, content: '' },
+  ]);
+
+  // Sign a session token
+  const sessionToken = sign({ userId, systemInstruction, voiceName, language }, getSessionSecret(), { expiresIn: ttl });
+
+  logger.info('[voice] session created', { userId, language, voiceName, ttl });
+
+  res.json({
+    sessionToken,
+    expiresAt: Math.floor(Date.now() / 1000) + ttl,
+  });
+});
+
+export { getSessionSecret };
+export default router;

api/src/routes/voice/relay.ts

@@ -0,0 +1,207 @@
+import WebSocket from 'ws';
+
+import { getGeminiLiveModel, getGeminiLiveWsUrl, getVoiceMaxDuration } from '../../ai/voice-model';
+import logger from '../../libs/logger';
+
+interface VoiceSession {
+  userId: string;
+  systemInstruction: string;
+  voiceName: string;
+  language: string;
+}
+
+/**
+ * GeminiLiveRelay — bridges a client WebSocket to the Gemini Live API.
+ *
+ * Protocol (client ↔ backend):
+ *   Client sends:  { type: "audio", data: "<base64 PCM 16kHz>" }
+ *                  { type: "end" }
+ *   Server sends:  { type: "ready" }
+ *                  { type: "audio", data: "<base64 PCM 24kHz>" }
+ *                  { type: "turnComplete" }
+ *                  { type: "error", message: "..." }
+ *                  { type: "closed" }
+ */
+export class GeminiLiveRelay {
+  private geminiWs: WebSocket | null = null;
+
+  private isSetupComplete = false;
+
+  private isClosed = false;
+
+  private maxDurationTimer: ReturnType<typeof setTimeout> | null = null;
+
+  constructor(
+    private clientWs: WebSocket,
+    private session: VoiceSession,
+  ) {}
+
+  /** Start the relay: connect to Gemini and wire up message handlers */
+  start(): void {
+    const wsUrl = getGeminiLiveWsUrl();
+    if (!wsUrl || wsUrl.endsWith('key=')) {
+      this.sendToClient({ type: 'error', message: 'Gemini API key not configured' });
+      this.close();
+      return;
+    }
+
+    logger.info('[voice-relay] connecting to Gemini Live', { userId: this.session.userId });
+
+    this.geminiWs = new WebSocket(wsUrl);
+
+    this.geminiWs.on('open', () => {
+      logger.info('[voice-relay] Gemini WS connected, sending setup');
+      this.sendSetup();
+    });
+
+    this.geminiWs.on('message', (data: WebSocket.Data) => {
+      this.handleGeminiMessage(data);
+    });
+
+    this.geminiWs.on('error', (error) => {
+      logger.error('[voice-relay] Gemini WS error', error);
+      this.sendToClient({ type: 'error', message: 'Connection to AI service failed' });
+      this.close();
+    });
+
+    this.geminiWs.on('close', (code, reason) => {
+      logger.info('[voice-relay] Gemini WS closed', { code, reason: reason.toString() });
+      if (!this.isClosed) {
+        this.sendToClient({ type: 'closed' });
+        this.close();
+      }
+    });
+
+    // Client message handler
+    this.clientWs.on('message', (data: WebSocket.Data) => {
+      this.handleClientMessage(data);
+    });
+
+    this.clientWs.on('close', () => {
+      logger.info('[voice-relay] client disconnected', { userId: this.session.userId });
+      this.close();
+    });
+
+    this.clientWs.on('error', (error) => {
+      logger.error('[voice-relay] client WS error', error);
+      this.close();
+    });
+
+    // Max duration safety timer
+    const maxDuration = getVoiceMaxDuration();
+    this.maxDurationTimer = setTimeout(() => {
+      logger.info('[voice-relay] max duration reached', { maxDuration, userId: this.session.userId });
+      this.sendToClient({ type: 'closed' });
+      this.close();
+    }, maxDuration * 1000);
+  }
+
+  /** Send the Gemini Live setup message with model config + system instruction */
+  private sendSetup(): void {
+    const model = getGeminiLiveModel();
+    const setup = {
+      setup: {
+        model: `models/${model}`,
+        generationConfig: {
+          responseModalities: ['AUDIO'],
+          speechConfig: {
+            voiceConfig: {
+              prebuiltVoiceConfig: { voiceName: this.session.voiceName },
+            },
+          },
+        },
+        systemInstruction: {
+          parts: [{ text: this.session.systemInstruction }],
+        },
+      },
+    };
+    this.geminiWs?.send(JSON.stringify(setup));
+  }
+
+  /** Handle messages from the client */
+  private handleClientMessage(raw: WebSocket.Data): void {
+    try {
+      const msg = JSON.parse(raw.toString());
+
+      if (msg.type === 'audio' && this.isSetupComplete) {
+        // Forward audio to Gemini in realtimeInput format
+        this.geminiWs?.send(
+          JSON.stringify({
+            realtimeInput: {
+              audio: {
+                data: msg.data,
+                mimeType: 'audio/pcm;rate=16000',
+              },
+            },
+          }),
+        );
+      } else if (msg.type === 'end') {
+        this.close();
+      }
+    } catch {
+      // Ignore malformed messages
+    }
+  }
+
+  /** Handle messages from Gemini Live API */
+  private handleGeminiMessage(raw: WebSocket.Data): void {
+    try {
+      const msg = JSON.parse(raw.toString());
+
+      // Setup complete
+      if (msg.setupComplete !== undefined) {
+        this.isSetupComplete = true;
+        logger.info('[voice-relay] setup complete', { userId: this.session.userId });
+        this.sendToClient({ type: 'ready' });
+        return;
+      }
+
+      const sc = msg.serverContent;
+      if (!sc) return;
+
+      // Forward audio data
+      if (sc.modelTurn?.parts) {
+        for (const part of sc.modelTurn.parts) {
+          const inline = part.inlineData;
+          if (inline?.data) {
+            this.sendToClient({ type: 'audio', data: inline.data });
+          }
+        }
+      }
+
+      // Turn complete
+      if (sc.turnComplete) {
+        this.sendToClient({ type: 'turnComplete' });
+      }
+    } catch {
+      // Ignore parse errors
+    }
+  }
+
+  /** Send a JSON message to the client WebSocket */
+  private sendToClient(msg: object): void {
+    if (this.clientWs.readyState === WebSocket.OPEN) {
+      this.clientWs.send(JSON.stringify(msg));
+    }
+  }
+
+  /** Clean up both connections */
+  close(): void {
+    if (this.isClosed) return;
+    this.isClosed = true;
+
+    if (this.maxDurationTimer) {
+      clearTimeout(this.maxDurationTimer);
+      this.maxDurationTimer = null;
+    }
+
+    if (this.geminiWs && this.geminiWs.readyState !== WebSocket.CLOSED) {
+      this.geminiWs.close();
+    }
+    this.geminiWs = null;
+
+    if (this.clientWs.readyState !== WebSocket.CLOSED) {
+      this.clientWs.close();
+    }
+  }
+}