feat(speech): add prompt voice input (NeuralNomadsAI#249)

## Summary
- add server-backed speech capabilities and transcription endpoints plus
UI settings for speech configuration
- add push-to-talk prompt voice input with microphone controls,
transcription insertion, and browser capability gating
- keep prompt controls aligned by restoring right-side nav placement and
moving the mic beside the expand control

Author: shantur

package-lock.json

@@ -4,6 +4,6 @@
   "private": true,
   "license": "MIT",
   "dependencies": {
-    "@opencode-ai/plugin": "1.2.24"
+    "@opencode-ai/plugin": "1.2.14"
   }
 }

packages/opencode-config/package.json

@@ -32,6 +32,7 @@
     "fastify": "^4.28.1",
     "fuzzysort": "^2.0.4",
     "node-forge": "^1.3.3",
+    "openai": "^6.27.0",
     "pino": "^9.4.0",
     "undici": "^6.19.8",
     "yaml": "^2.4.2",

packages/server/package.json

@@ -207,6 +207,36 @@ export interface BinaryValidationResult {
   error?: string
 }
 
+export interface SpeechSegment {
+  startMs: number
+  endMs: number
+  text: string
+}
+
+export interface SpeechCapabilitiesResponse {
+  available: boolean
+  configured: boolean
+  provider: string
+  supportsStt: boolean
+  supportsTts: boolean
+  baseUrl?: string
+  sttModel: string
+  ttsModel: string
+  ttsVoice: string
+}
+
+export interface SpeechTranscriptionResponse {
+  text: string
+  language?: string
+  durationMs?: number
+  segments?: SpeechSegment[]
+}
+
+export interface SpeechSynthesisResponse {
+  audioBase64: string
+  mimeType: string
+}
+
 export type WorkspaceEventType =
   | "workspace.created"
   | "workspace.started"

packages/server/src/api-types.ts

@@ -23,6 +23,7 @@ import { AuthManager, BOOTSTRAP_TOKEN_STDOUT_PREFIX, DEFAULT_AUTH_USERNAME } fro
 import { resolveHttpsOptions } from "./server/tls"
 import { resolveNetworkAddresses } from "./server/network-addresses"
 import { startDevReleaseMonitor } from "./releases/dev-release-monitor"
+import { SpeechService } from "./speech/service"
 
 const require = createRequire(import.meta.url)
 
@@ -304,6 +305,7 @@ async function main() {
   })
   const fileSystemBrowser = new FileSystemBrowser({ rootDir: options.rootDir, unrestricted: options.unrestrictedRoot })
   const instanceStore = new InstanceStore(configLocation.instancesDir)
+  const speechService = new SpeechService(settings, logger.child({ component: "speech" }))
   const instanceEventBridge = new InstanceEventBridge({
     workspaceManager,
     eventBus,
@@ -388,6 +390,7 @@ async function main() {
         eventBus,
         serverMeta,
         instanceStore,
+        speechService,
         authManager,
         uiStaticDir: uiResolution.uiStaticDir ?? DEFAULT_UI_STATIC_DIR,
         uiDevServerUrl: uiResolution.uiDevServerUrl,
@@ -408,6 +411,7 @@ async function main() {
         eventBus,
         serverMeta,
         instanceStore,
+        speechService,
         authManager,
         uiStaticDir: uiResolution.uiStaticDir ?? DEFAULT_UI_STATIC_DIR,
         uiDevServerUrl: undefined,

packages/server/src/index.ts

@@ -21,12 +21,14 @@ import { registerStorageRoutes } from "./routes/storage"
 import { registerPluginRoutes } from "./routes/plugin"
 import { registerBackgroundProcessRoutes } from "./routes/background-processes"
 import { registerWorktreeRoutes } from "./routes/worktrees"
+import { registerSpeechRoutes } from "./routes/speech"
 import { ServerMeta } from "../api-types"
 import { InstanceStore } from "../storage/instance-store"
 import { BackgroundProcessManager } from "../background-processes/manager"
 import type { AuthManager } from "../auth/manager"
 import { registerAuthRoutes } from "./routes/auth"
 import { sendUnauthorized, wantsHtml } from "../auth/http-auth"
+import type { SpeechService } from "../speech/service"
 
 interface HttpServerDeps {
   bindHost: string
@@ -41,6 +43,7 @@ interface HttpServerDeps {
   eventBus: EventBus
   serverMeta: ServerMeta
   instanceStore: InstanceStore
+  speechService: SpeechService
   authManager: AuthManager
   uiStaticDir: string
   uiDevServerUrl?: string
@@ -252,6 +255,7 @@ export function createHttpServer(deps: HttpServerDeps) {
     eventBus: deps.eventBus,
     workspaceManager: deps.workspaceManager,
   })
+  registerSpeechRoutes(app, { speechService: deps.speechService })
   registerPluginRoutes(app, { workspaceManager: deps.workspaceManager, eventBus: deps.eventBus, logger: proxyLogger })
   registerBackgroundProcessRoutes(app, { backgroundProcessManager })
   registerInstanceProxyRoutes(app, { workspaceManager: deps.workspaceManager, logger: proxyLogger })

packages/server/src/server/http-server.ts

@@ -3,6 +3,7 @@ import { z } from "zod"
 import { probeBinaryVersion } from "../../workspaces/runtime"
 import type { SettingsService } from "../../settings/service"
 import type { Logger } from "../../logger"
+import { sanitizeConfigDoc, sanitizeConfigOwner } from "../../settings/public-config"
 
 interface RouteDeps {
   settings: SettingsService
@@ -20,23 +21,26 @@ function validateBinaryPath(binaryPath: string): { valid: boolean; version?: str
 
 export function registerSettingsRoutes(app: FastifyInstance, deps: RouteDeps) {
   // Full-document access
-  app.get("/api/storage/config", async () => deps.settings.getDoc("config"))
+  app.get("/api/storage/config", async () => sanitizeConfigDoc(deps.settings.getDoc("config")))
   app.patch("/api/storage/config", async (request, reply) => {
     try {
-      return deps.settings.mergePatchDoc("config", request.body ?? {})
+      return sanitizeConfigDoc(deps.settings.mergePatchDoc("config", request.body ?? {}))
     } catch (error) {
       reply.code(400)
       return { error: error instanceof Error ? error.message : "Invalid patch" }
     }
   })
 
   app.get<{ Params: { owner: string } }>("/api/storage/config/:owner", async (request) => {
-    return deps.settings.getOwner("config", request.params.owner)
+    return sanitizeConfigOwner(request.params.owner, deps.settings.getOwner("config", request.params.owner))
   })
 
   app.patch<{ Params: { owner: string } }>("/api/storage/config/:owner", async (request, reply) => {
     try {
-      return deps.settings.mergePatchOwner("config", request.params.owner, request.body ?? {})
+      return sanitizeConfigOwner(
+        request.params.owner,
+        deps.settings.mergePatchOwner("config", request.params.owner, request.body ?? {}),
+      )
     } catch (error) {
       reply.code(400)
       return { error: error instanceof Error ? error.message : "Invalid patch" }

packages/server/src/server/routes/settings.ts

@@ -0,0 +1,60 @@
+import type { FastifyInstance } from "fastify"
+import { z } from "zod"
+import type { SpeechService } from "../../speech/service"
+
+interface RouteDeps {
+  speechService: SpeechService
+}
+
+const TranscribeBodySchema = z.object({
+  audioBase64: z.string().min(1, "Audio payload is required"),
+  mimeType: z.string().min(1, "Audio MIME type is required"),
+  filename: z.string().optional(),
+  language: z.string().optional(),
+  prompt: z.string().optional(),
+})
+
+const SynthesizeBodySchema = z.object({
+  text: z.string().trim().min(1, "Text is required"),
+  format: z.enum(["mp3", "wav", "opus"]).optional(),
+})
+
+function getSpeechErrorStatus(error: unknown): number {
+  if (error instanceof z.ZodError) {
+    return 400
+  }
+  if (error instanceof Error && /not configured/i.test(error.message)) {
+    return 503
+  }
+  return 502
+}
+
+function getSpeechErrorMessage(error: unknown, fallback: string): string {
+  return error instanceof Error ? error.message : fallback
+}
+
+export function registerSpeechRoutes(app: FastifyInstance, deps: RouteDeps) {
+  app.get("/api/speech/capabilities", async () => deps.speechService.getCapabilities())
+
+  app.post("/api/speech/transcribe", async (request, reply) => {
+    try {
+      const body = TranscribeBodySchema.parse(request.body ?? {})
+      return await deps.speechService.transcribe(body)
+    } catch (error) {
+      request.log.error({ err: error }, "Failed to transcribe audio")
+      reply.code(getSpeechErrorStatus(error))
+      return { error: getSpeechErrorMessage(error, "Failed to transcribe audio") }
+    }
+  })
+
+  app.post("/api/speech/synthesize", async (request, reply) => {
+    try {
+      const body = SynthesizeBodySchema.parse(request.body ?? {})
+      return await deps.speechService.synthesize(body)
+    } catch (error) {
+      request.log.error({ err: error }, "Failed to synthesize audio")
+      reply.code(getSpeechErrorStatus(error))
+      return { error: getSpeechErrorMessage(error, "Failed to synthesize audio") }
+    }
+  })
+}

packages/server/src/server/routes/speech.ts

@@ -0,0 +1,40 @@
+import type { SettingsDoc } from "./yaml-doc-store"
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}
+
+function sanitizeServerOwner(value: SettingsDoc): SettingsDoc {
+  const next: SettingsDoc = { ...value }
+  const speech = isPlainObject(next.speech) ? { ...next.speech } : null
+
+  if (!speech) {
+    return next
+  }
+
+  const rawApiKey = typeof speech.apiKey === "string" ? speech.apiKey.trim() : ""
+  if (rawApiKey) {
+    delete speech.apiKey
+    speech.hasApiKey = true
+  } else if (!("hasApiKey" in speech)) {
+    speech.hasApiKey = false
+  }
+
+  next.speech = speech
+  return next
+}
+
+export function sanitizeConfigOwner(owner: string, value: SettingsDoc): SettingsDoc {
+  if (owner !== "server") {
+    return value
+  }
+  return sanitizeServerOwner(value)
+}
+
+export function sanitizeConfigDoc(value: SettingsDoc): SettingsDoc {
+  const next: SettingsDoc = { ...value }
+  if (isPlainObject(next.server)) {
+    next.server = sanitizeServerOwner(next.server)
+  }
+  return next
+}

packages/server/src/settings/public-config.ts

@@ -4,6 +4,7 @@ import type { ConfigLocation } from "../config/location"
 import { YamlDocStore, type SettingsDoc } from "./yaml-doc-store"
 import { migrateSettingsLayout } from "./migrate"
 import type { WorkspaceEventPayload } from "../api-types"
+import { sanitizeConfigOwner } from "./public-config"
 
 export type DocKind = "config" | "state"
 
@@ -45,10 +46,11 @@ export class SettingsService {
   private publish(kind: DocKind, owner: string, value?: SettingsDoc) {
     if (!this.eventBus) return
     const type = kind === "config" ? "storage.configChanged" : "storage.stateChanged"
+    const nextValue = value ?? this.getOwner(kind, owner)
     const payload: WorkspaceEventPayload = {
       type,
       owner,
-      value: value ?? this.getOwner(kind, owner),
+      value: kind === "config" ? sanitizeConfigOwner(owner, nextValue) : nextValue,
     } as any
     this.eventBus.publish(payload)
   }