diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 2ba0c353a..a02b51ff7 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -601,6 +601,7 @@ dependencies = [ "objc2-app-kit", "objc2-avf-audio", "objc2-core-audio", + "objc2-core-foundation", "objc2-foundation", "ort", "ort-sys", diff --git a/src-tauri/crates/berd-call/Cargo.toml b/src-tauri/crates/berd-call/Cargo.toml index d02f9ca5f..dac0f6006 100644 --- a/src-tauri/crates/berd-call/Cargo.toml +++ b/src-tauri/crates/berd-call/Cargo.toml @@ -66,6 +66,7 @@ objc2 = "0.6" objc2-app-kit = "0.3.2" objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2"] } objc2-core-audio = "0.3.2" +objc2-core-foundation = "0.3.2" objc2-foundation = { version = "0.3.2", features = ["NSNotification", "NSOperation", "NSString", "block2"] } [target.'cfg(target_os = "macos")'.build-dependencies] diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index d38454cab..2f307a063 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -58,12 +58,12 @@ initial policy; a host-specific `auto` mode must be resolved before the request. {"type":"hello","id":1,"input_during_tts":"allow_barge_in","status_sound_output_device":"MacBook Pro Speakers"} ``` -The response uses `protocol:5` as the exact session message-set version. The +The response uses `protocol:7` as the exact session message-set version. The parent must reject a version it does not support; the JSON version is independent of the fixed binary framing marker described below: ```json -{"type":"ready","id":1,"protocol":5,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} +{"type":"ready","id":1,"protocol":7,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} ``` The `session.tts` object is the authoritative, sanitized TTS configuration. @@ -397,3 +397,12 @@ receipts, flushes, and exits. EOF, malformed framing, fatal input failure, or process death cancels both authorities without transparent restart. A fatal error is flushed exactly once and followed by no protocol output. + + +## Local speech detection (protocol 7) + +Conventional session readiness includes an optional `session.detection` snapshot with `settings.listening` and `settings.interruption` (`value`, `dynamic`), plus `adjustments`, `no_result` and `missed_detection` counters. These local controls are separate from Realtime Spokesperson turn detection. + +A host sends `{"type":"set_detection_settings","id":42,"settings":{"listening":{"value":0.7,"dynamic":true},"interruption":{"value":0.8,"dynamic":false}}}` to replace both settings atomically and discard stale input/learning evidence. The session returns `detection_settings_result` with that ID and an applied `snapshot`, or `snapshot:null` and a `message` explaining rejection. Invalid settings leave the current values unchanged. Dynamic choices require macOS STT. + +During adaptive tuning, the session sends `detection_settings_result` with `id:null` and the new snapshot. Hosts must handle these unsolicited updates, report applied values, and keep profile persistence associated with their actual input/output device pair. Protocol 7 adds `settings.timing` (`onset_ms`, `release_ms`) for Apple local VAD. Older profiles omit this field and receive defaults of 16 ms onset and 300 ms release. Final transcripts are emitted immediately, independently of the local activity gate. diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index ee08673f8..3b654920b 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -9,6 +9,27 @@ management, synthesis and benchmark tools described below. Desktop Berd keeps its persisted settings and user interface. The standalone command includes a small macOS host for default-device capture, playback, and transcript delivery. +### Local speech detection + +Conventional calls accept independent `--listening-threshold 0..1` (default `0.5`) and `--interruption-threshold 0..1` (default `0.65`) at `start` or `session`. Higher values require stronger evidence of speech. Listening controls detection before agent playback, including the gate that holds replies while input is speaking or awaiting recognition. Interruption controls detection during agent playback. These local thresholds do not change provider endpointing or Spokesperson server VAD. + +On macOS, a local speech detection also marks recognition pending. After local speech ends, admission continues waiting for recognition to settle, with a five-second no-result fallback. Raising the listening threshold can reduce false starts in background noise; it does not shorten that fallback or prove a particular microphone's false detections are resolved. + +Device profiles retain both values independently; legacy saved call arguments are accepted during migration. To change a running call, supply its complete session configuration, for example `settings --restart --voice Aaron --language en-US --listening-threshold 0.8 --interruption-threshold 0.7`. Restart replaces the session options; include the existing backend, model, rate and other choices you want to retain. + +### Experimental dynamic tuning and audio profiles + +With macOS speech recognition, `--dynamic-listening true` and `--dynamic-interruption true` independently opt into outcome-based tuning. Both default to false. Each enabled threshold moves one 0.01 tick after each qualifying unmatched outcome, without a cooldown. Dynamic values start and remain within 0.10 to 0.95; manual thresholds still accept 0 to 1. + +Apple recognition also supports per-audio-setup `timing` settings through `detection-profile --settings`: `onset_ms` (16 to 1000, default 16) and `release_ms` (16 to 3000, default 300). Timings round up to 16 ms audio frames and apply only to local activity detection. Finalized transcripts are forwarded immediately, independently of release timing. The recognizer continues to receive all microphone audio during onset gating. Other recognition backends retain their existing segmentation timing. + + +A local detection that settles without a nonempty transcript suggests raising sensitivity's threshold. A nonempty transcript after at least five seconds without local detection suggests lowering it. Recognized speech clears false-trigger evidence. Split-result grace periods, mode changes, mute/reset epochs and device changes discard ambiguous evidence. These are experimental heuristics: recognition failure and playback leakage can still mislead them. They do not establish an optimal threshold or replace acoustic echo cancellation. + +The standalone macOS host saves applied thresholds and dynamic choices in `detection-profiles.json` beside its saved settings, keyed by the stable input/output device UID pair. It restores the pair on call startup and checks for device changes once per second. Explicit threshold or dynamic flags override the matching profile choice; unspecified choices retain their saved values. Detection flags are removed from inherited session arguments so a later voice-only restart restores learned values. A new pair starts with defaults during a call. Profile files contain settings, not audio recordings or transcript text. An explicit profile edit takes precedence over a late automatic update. + +`berd-call detection-profile` reports the default audio setup and saved choices without starting capture. `berd-call detection-profile --settings '{"listening":{"value":0.70,"dynamic":true},"interruption":{"value":0.80,"dynamic":false}}'` saves choices for that pair. During a call, `berd-call status` exposes applied values and evidence counters in `session.detection`, plus `audioSetup`. Saving a profile alone affects its next restoration; the plugin restarts the call session when a user changes a control. Dynamic adjustments apply during the current session. + ## Command-line interface On macOS, install [Berd](https://github.com/block/berd/releases) first. Its diff --git a/src-tauri/crates/berd-call/VAD_VISION.md b/src-tauri/crates/berd-call/VAD_VISION.md new file mode 100644 index 000000000..216a8aca9 --- /dev/null +++ b/src-tauri/crates/berd-call/VAD_VISION.md @@ -0,0 +1,11 @@ +# Adaptive speech detection + +Proposed direction, October 22, 2026. Independent static thresholds are implemented by PR 421. Adaptive tuning and audio profiles below are proposed until their implementation is validated. + +A caller should be able to stop background noise holding replies without losing quiet speech. Listening and interruption each have a slider, visible current value, and an opt-in Dynamically adjust checkbox. Adjustments move one 0.01 tick at a time within 0.10 to 0.95. Manual values remain available across the full 0 to 1 range. + +For the initial experimental macOS recognizer, local detections that settle without a nonempty transcript are evidence to increase the threshold. Nonempty transcripts without corresponding local detection provide evidence to decrease it. These are heuristics, not ground truth: recognizer failures, late or split results and playback leakage can mislead them. Ignore ambiguous observations across mute/reset, playback-mode and device changes; move one tick per qualifying outcome and expose evidence counters with current values. Other recognition backends require their own reliable outcome signals before enabling adaptive tuning. + +Save independent values and dynamic choices by stable input/output device identity pair. AirPods input/output, another headset, and the built-in input/output pair retain separate profiles. Restoring a profile must discard old learning evidence so one setup cannot train another. No microphone recordings or transcript text are stored in these profiles. + +The host owns device identity and persistence. The shared input runtime owns thresholds, and the host-facing session protocol carries updates and reports applied values. Existing settings files remain readable by older binaries. Prefer a separate bounded, atomically replaced profile file to changing their strict schema. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 68e43ce2d..a3772424a 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -147,13 +147,21 @@ Usage: --model-dir ABSOLUTE_PATH --voice ID Each form also accepts: - [--rate FLOAT] + [--rate FLOAT] [--listening-threshold 0..1] [--interruption-threshold 0..1] + [--dynamic-listening true|false] [--dynamic-interruption true|false] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] [--stt-hint PHRASE]... [--mode conventional|expert-spokesperson] [--allow-direct-agent-speech] [--spokesperson-voice NAME] [--spokesperson-rate 0.25..1.5] [--spokesperson-options JSON] [--realtime-url WS_URL] [--stt-url WS_URL] [--tts-url HTTP_URL] +--listening-threshold sets local speech detection before agent playback (default 0.5). +--interruption-threshold sets local speech detection during agent playback (default 0.65). +Higher values require stronger evidence of speech before pausing playback. +Dynamic adjustment is experimental, off by default, and requires macOS STT. +Each enabled threshold learns independently, moving by 0.01 within 0.10..0.95. +Use detection-profile [--settings JSON] to read or save the default audio setup profile. + The host owns microphone capture, audio playback, transcript delivery, and agent integration. See PROTOCOL.md for the framed stdin, stdout, and PCM contracts. Endpoint URL flags override only the matching service for this call diff --git a/src-tauri/crates/berd-call/src/detection.rs b/src-tauri/crates/berd-call/src/detection.rs new file mode 100644 index 000000000..c1c8106b0 --- /dev/null +++ b/src-tauri/crates/berd-call/src/detection.rs @@ -0,0 +1,432 @@ +//! Experimental outcome-based tuning of independent local speech thresholds. +use serde::{Deserialize, Serialize}; +use std::time::{Duration, Instant}; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct Threshold { + pub value: f32, + #[serde(default)] + pub dynamic: bool, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DetectionSettings { + pub listening: Threshold, + pub interruption: Threshold, + #[serde(default)] + pub timing: VadTiming, +} + +/// Apple recognition keeps receiving audio while these local activity gates settle. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct VadTiming { + pub onset_ms: u32, + pub release_ms: u32, +} + +impl Default for VadTiming { + fn default() -> Self { + Self { + onset_ms: 16, + release_ms: 300, + } + } +} + +#[derive(Default)] +pub struct SpeechGate { + active: bool, + speech_frames: u32, + silence_frames: u32, +} + +impl SpeechGate { + pub fn observe(&mut self, detected: bool, timing: VadTiming) -> bool { + if detected { + self.silence_frames = 0; + self.speech_frames = self.speech_frames.saturating_add(1); + if self.speech_frames >= timing.onset_ms.div_ceil(16).max(1) { + self.active = true; + } + } else { + self.speech_frames = 0; + self.silence_frames = self.silence_frames.saturating_add(1); + if self.silence_frames >= timing.release_ms.div_ceil(16).max(1) { + self.active = false; + } + } + self.active + } +} + +pub fn is_detection_flag(flag: &str) -> bool { + matches!( + flag, + "--listening-threshold" + | "--interruption-threshold" + | "--dynamic-listening" + | "--dynamic-interruption" + ) +} + +pub fn without_detection_arguments(arguments: &[String]) -> Vec { + filter_arguments(arguments, false) +} + +pub fn detection_arguments(arguments: &[String]) -> Vec { + filter_arguments(arguments, true) +} + +fn filter_arguments(arguments: &[String], detection: bool) -> Vec { + let mut result = Vec::new(); + let mut index = 0; + while index < arguments.len() { + let flag = &arguments[index]; + let count = if flag == "session" || flag == "--allow-direct-agent-speech" { + 1 + } else { + 2 + }; + let end = (index + count).min(arguments.len()); + if is_detection_flag(flag) == detection { + result.extend_from_slice(&arguments[index..end]); + } + index = end; + } + result +} + +impl Default for DetectionSettings { + fn default() -> Self { + Self { + timing: VadTiming::default(), + listening: Threshold { + value: 0.5, + dynamic: false, + }, + interruption: Threshold { + value: 0.65, + dynamic: false, + }, + } + } +} + +impl DetectionSettings { + pub fn validate(self) -> Result { + if !(16..=1000).contains(&self.timing.onset_ms) + || !(16..=3000).contains(&self.timing.release_ms) + { + return Err("VAD onset must be 16 to 1000 ms and release 16 to 3000 ms".into()); + } + for threshold in [self.listening, self.interruption] { + if !threshold.value.is_finite() || !(0.0..=1.0).contains(&threshold.value) { + return Err("speech thresholds must be finite and between 0 and 1".into()); + } + if threshold.dynamic && !(0.10..=0.95).contains(&threshold.value) { + return Err("dynamic speech thresholds must start between 0.10 and 0.95".into()); + } + } + Ok(self) + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +pub struct DetectionSnapshot { + pub settings: DetectionSettings, + pub adjustments: u64, + pub no_result: u64, + pub missed_detection: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DetectionMode { + Listening, + Interruption, +} + +const STEP: f32 = 0.01; +const RESULT_GRACE: Duration = Duration::from_secs(5); + +pub struct DetectionTuner { + snapshot: DetectionSnapshot, + window: Option, + grace_until: Option, +} + +struct Observation { + mode: DetectionMode, + began: Instant, + detected: bool, + ambiguous: bool, +} + +impl DetectionTuner { + pub fn new(settings: DetectionSettings) -> Self { + Self { + snapshot: DetectionSnapshot { + settings, + adjustments: 0, + no_result: 0, + missed_detection: 0, + }, + window: None, + grace_until: None, + } + } + pub fn snapshot(&self) -> DetectionSnapshot { + self.snapshot + } + pub fn observe(&mut self, mode: DetectionMode, detected: bool, now: Instant) { + if self + .window + .as_ref() + .is_some_and(|window| window.mode != mode && !window.detected) + { + self.window = None; + } + let window = self.window.get_or_insert(Observation { + mode, + began: now, + detected: false, + ambiguous: false, + }); + window.ambiguous |= window.mode != mode; + window.detected |= detected; + } + pub fn final_transcript(&mut self, now: Instant) { + if let Some(window) = self.window.take().filter(|window| !window.ambiguous) { + if !window.detected + && now.duration_since(window.began) >= RESULT_GRACE + && self.grace_until.is_none_or(|until| now >= until) + { + self.snapshot.missed_detection += 1; + self.adjust(window.mode, false); + } + } + self.grace_until = Some(now + RESULT_GRACE); + } + pub fn no_result(&mut self, _now: Instant) { + if let Some(window) = self + .window + .take() + .filter(|window| window.detected && !window.ambiguous) + { + self.snapshot.no_result += 1; + self.adjust(window.mode, true); + } + } + fn adjust(&mut self, mode: DetectionMode, increase: bool) { + let threshold = match mode { + DetectionMode::Listening => &mut self.snapshot.settings.listening, + DetectionMode::Interruption => &mut self.snapshot.settings.interruption, + }; + if !threshold.dynamic { + return; + } + // Round to slider ticks, and never jump a manual value toward a bound. + let next = + ((threshold.value + if increase { STEP } else { -STEP }) * 100.0).round() / 100.0; + if !(0.10..=0.95).contains(&next) { + return; + } + threshold.value = next; + self.snapshot.adjustments += 1; + } + pub fn reset_evidence(&mut self) { + self.window = None; + self.grace_until = None; + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn gate_requires_consecutive_onset_and_release_frames() { + let mut gate = SpeechGate::default(); + let timing = VadTiming { + onset_ms: 32, + release_ms: 48, + }; + assert!(!gate.observe(true, timing)); + assert!(!gate.observe(false, timing)); + assert!(!gate.observe(true, timing)); + assert!(gate.observe(true, timing)); + assert!(gate.observe(false, timing)); + assert!(gate.observe(false, timing)); + assert!(!gate.observe(false, timing)); + } + + #[test] + fn legacy_profiles_get_default_timing_and_invalid_timing_is_rejected() { + let settings: DetectionSettings = + serde_json::from_str(r#"{"listening":{"value":0.5},"interruption":{"value":0.65}}"#) + .unwrap(); + assert_eq!(settings.timing, VadTiming::default()); + for timing in [ + VadTiming { + onset_ms: 0, + release_ms: 300, + }, + VadTiming { + onset_ms: 16, + release_ms: 3001, + }, + ] { + assert!(DetectionSettings { timing, ..settings }.validate().is_err()); + } + } + + fn tuner() -> DetectionTuner { + DetectionTuner::new(DetectionSettings { + timing: VadTiming::default(), + listening: Threshold { + value: 0.5, + dynamic: true, + }, + interruption: Threshold { + value: 0.65, + dynamic: true, + }, + }) + } + #[test] + fn detection_flags_do_not_consume_flag_shaped_hint_values() { + let arguments = [ + "session", + "--stt-hint", + "--listening-threshold", + "--language", + "en-US", + "--listening-threshold", + "0.8", + "--allow-direct-agent-speech", + ] + .map(str::to_string); + assert_eq!( + detection_arguments(&arguments), + ["--listening-threshold", "0.8"] + ); + assert_eq!( + without_detection_arguments(&arguments), + [ + "session", + "--stt-hint", + "--listening-threshold", + "--language", + "en-US", + "--allow-direct-agent-speech" + ] + ); + } + + #[test] + fn each_no_result_moves_only_the_detecting_mode_one_tick() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..3 { + let time = now + Duration::from_secs(n * 10); + tuner.observe(DetectionMode::Listening, true, time); + tuner.no_result(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings.listening.value, 0.53); + assert_eq!(tuner.snapshot().settings.interruption.value, 0.65); + assert_eq!(tuner.snapshot().adjustments, 3); + } + #[test] + fn idle_mode_changes_do_not_disable_interruption_learning() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..3 { + let time = now + Duration::from_secs(n * 40); + tuner.observe(DetectionMode::Listening, false, time); + tuner.observe( + DetectionMode::Interruption, + true, + time + Duration::from_secs(1), + ); + tuner.no_result(time + Duration::from_secs(7)); + } + assert_eq!(tuner.snapshot().settings.interruption.value, 0.68); + assert_eq!(tuner.snapshot().settings.listening.value, 0.5); + } + + #[test] + fn manual_mode_never_learns() { + let settings = DetectionSettings::default(); + let mut tuner = DetectionTuner::new(settings); + let now = Instant::now(); + for n in 0..20 { + let time = now + Duration::from_secs(n * 40); + tuner.observe(DetectionMode::Interruption, true, time); + tuner.no_result(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings, settings); + } + #[test] + fn transcripts_without_detection_lower_only_the_observed_mode() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..3 { + let time = now + Duration::from_secs(n * 40); + tuner.observe(DetectionMode::Interruption, false, time); + tuner.final_transcript(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings.interruption.value, 0.62); + assert_eq!(tuner.snapshot().settings.listening.value, 0.5); + } + #[test] + fn adjustments_have_no_cooldown_and_stay_inside_dynamic_bounds() { + let mut tuner = tuner(); + tuner.snapshot.settings.listening.value = 0.94; + let now = Instant::now(); + for n in 0..100 { + let time = now + Duration::from_secs(n * 7); + tuner.observe(DetectionMode::Listening, true, time); + tuner.no_result(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings.listening.value, 0.95); + assert_eq!(tuner.snapshot().adjustments, 1); + tuner.snapshot.settings.listening.value = 0.5; + tuner.reset_evidence(); + for n in 0..6 { + let time = now + Duration::from_secs(1000 + n); + tuner.observe(DetectionMode::Listening, true, time); + tuner.no_result(time); + } + assert_eq!(tuner.snapshot().adjustments, 7); + assert_eq!(tuner.snapshot().settings.listening.value, 0.56); + } + + #[test] + fn recent_split_finals_and_detected_speech_do_not_lower_thresholds() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..20 { + let time = now + Duration::from_secs(n); + tuner.observe(DetectionMode::Listening, n == 0, time); + tuner.final_transcript(time); + } + assert_eq!(tuner.snapshot().adjustments, 0); + } + + #[test] + fn mode_switch_and_reset_discard_ambiguous_evidence() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..10 { + let time = now + Duration::from_secs(n * 40); + tuner.observe(DetectionMode::Listening, true, time); + tuner.observe(DetectionMode::Interruption, false, time); + tuner.no_result(time + Duration::from_secs(6)); + tuner.observe(DetectionMode::Listening, true, time); + tuner.reset_evidence(); + tuner.no_result(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings, self::tuner().snapshot().settings); + } +} diff --git a/src-tauri/crates/berd-call/src/detection_profiles.rs b/src-tauri/crates/berd-call/src/detection_profiles.rs new file mode 100644 index 000000000..f7c5279ed --- /dev/null +++ b/src-tauri/crates/berd-call/src/detection_profiles.rs @@ -0,0 +1,210 @@ +use berd_call::{detection::DetectionSettings, macos_audio_route::AudioSetup}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +#[derive(Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Profiles { + profiles: BTreeMap, +} + +pub fn settings_for_start( + persisted: Option, + configured: DetectionSettings, + arguments: &[String], +) -> Result { + let mut settings = persisted.unwrap_or(configured); + for pair in berd_call::detection::detection_arguments(arguments).chunks_exact(2) { + match pair[0].as_str() { + "--listening-threshold" => settings.listening.value = configured.listening.value, + "--interruption-threshold" => { + settings.interruption.value = configured.interruption.value + } + "--dynamic-listening" => settings.listening.dynamic = configured.listening.dynamic, + "--dynamic-interruption" => { + settings.interruption.dynamic = configured.interruption.dynamic + } + _ => {} + } + } + settings.validate() +} + +pub fn path() -> Result { + Ok(crate::saved_settings::path()?.with_file_name("detection-profiles.json")) +} + +fn key(setup: &AudioSetup) -> String { + serde_json::to_string(&[&setup.input_uid, &setup.output_uid]).expect("device strings serialize") +} + +fn read(path: &Path) -> Result { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Profiles::default()) + } + Err(error) => return Err(error.to_string()), + }; + let mut bytes = Vec::new(); + file.take(65537) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + if bytes.len() > 65536 { + return Err("speech profiles exceed 64 KiB".into()); + } + let profiles: Profiles = serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + if profiles.profiles.len() > 64 { + return Err("too many speech profiles".into()); + } + for settings in profiles.profiles.values() { + settings.validate()?; + } + Ok(profiles) +} + +pub fn load(path: &Path, setup: &AudioSetup) -> Result, String> { + Ok(read(path)?.profiles.get(&key(setup)).copied()) +} + +pub fn save(path: &Path, setup: &AudioSetup, settings: DetectionSettings) -> Result<(), String> { + write(path, setup, settings, None).map(|_| ()) +} + +pub fn save_if_unchanged( + path: &Path, + setup: &AudioSetup, + expected: Option, + settings: DetectionSettings, +) -> Result { + write(path, setup, settings, Some(expected)) +} + +fn write( + path: &Path, + setup: &AudioSetup, + settings: DetectionSettings, + expected: Option>, +) -> Result { + settings.validate()?; + let parent = path + .parent() + .ok_or("speech profiles need a parent directory")?; + std::fs::create_dir_all(parent).map_err(|error| error.to_string())?; + let lock = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path.with_extension("json.lock")) + .map_err(|error| error.to_string())?; + fs2::FileExt::lock_exclusive(&lock).map_err(|error| error.to_string())?; + let mut profiles = read(path)?; + if expected.is_some_and(|expected| profiles.profiles.get(&key(setup)).copied() != expected) { + return Ok(false); + } + if profiles.profiles.get(&key(setup)) == Some(&settings) { + return Ok(true); + } + profiles.profiles.insert(key(setup), settings); + if profiles.profiles.len() > 64 { + return Err("too many speech profiles".into()); + } + let bytes = serde_json::to_vec_pretty(&profiles).map_err(|error| error.to_string())?; + if bytes.len() > 65536 { + return Err("speech profiles exceed 64 KiB".into()); + } + let mut file = tempfile::NamedTempFile::new_in(parent).map_err(|error| error.to_string())?; + file.write_all(&bytes) + .and_then(|()| file.as_file().sync_all()) + .map_err(|error| error.to_string())?; + file.persist(path).map_err(|error| error.to_string())?; + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + fn setup(input: &str, output: &str) -> AudioSetup { + AudioSetup { + input_uid: input.into(), + output_uid: output.into(), + input_name: "same display name".into(), + output_name: "same display name".into(), + } + } + #[test] + fn explicit_start_and_restart_flags_override_only_their_profile_choices() { + let mut profile = DetectionSettings::default(); + profile.listening.value = 0.73; + profile.listening.dynamic = true; + profile.interruption.value = 0.82; + let mut configured = DetectionSettings::default(); + configured.listening.value = 0.8; + configured.interruption.dynamic = true; + let arguments = [ + "session", + "--listening-threshold", + "0.8", + "--dynamic-interruption", + "true", + ] + .map(str::to_string); + let restored = settings_for_start(Some(profile), configured, &arguments).unwrap(); + assert_eq!(restored.listening.value, 0.8); + assert!(restored.listening.dynamic); + assert_eq!(restored.interruption.value, 0.82); + assert!(restored.interruption.dynamic); + let inherited = berd_call::detection::without_detection_arguments(&arguments); + assert_eq!(inherited, ["session"]); + assert_eq!( + settings_for_start(Some(profile), configured, &inherited).unwrap(), + profile + ); + } + + #[test] + fn late_learning_cannot_overwrite_an_explicit_profile_edit() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("profiles.json"); + let setup = setup("mic", "output"); + let original = DetectionSettings::default(); + save(&path, &setup, original).unwrap(); + let mut edited = original; + edited.listening.value = 0.77; + save(&path, &setup, edited).unwrap(); + let mut learned = original; + learned.listening.value = 0.51; + assert!(!save_if_unchanged(&path, &setup, Some(original), learned).unwrap()); + assert_eq!(load(&path, &setup).unwrap(), Some(edited)); + assert!(save_if_unchanged(&path, &setup, Some(edited), learned).unwrap()); + } + + #[test] + fn profiles_are_isolated_by_both_stable_device_ids_and_survive_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("profiles.json"); + let airpods = setup("airpods-mic", "airpods-output"); + let mixed = setup("mac-mic", "airpods-output"); + let mut first = DetectionSettings::default(); + first.listening.value = 0.71; + first.listening.dynamic = true; + let mut second = DetectionSettings::default(); + second.interruption.value = 0.83; + save(&path, &airpods, first).unwrap(); + save(&path, &mixed, second).unwrap(); + assert_eq!(load(&path, &airpods).unwrap(), Some(first)); + assert_eq!(load(&path, &mixed).unwrap(), Some(second)); + assert_eq!( + load(&path, &setup("headset-mic", "headset-output")).unwrap(), + None + ); + assert!(!std::fs::read_to_string(path) + .unwrap() + .contains("same display name")); + } +} diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 3b9ce4735..d998756ff 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -418,6 +418,7 @@ fn start_session( .store(muted.load(Ordering::SeqCst), Ordering::SeqCst); actor.capture_muted = muted; actor.capture_mute_epoch = mute_epoch; + actor.restore_detection_profile(&arguments)?; actor.system_input_mute = system_input_mute; if actor.system_input_mute.is_none() { match crate::system_input_mute::SystemInputMute::install( @@ -467,7 +468,7 @@ fn start_session( || arguments .iter() .any(|argument| argument == crate::DIRECT_AGENT_SPEECH), - session_arguments: arguments, + session_arguments: berd_call::detection::without_detection_arguments(&arguments), }); Ok(StartedSession { child, @@ -891,6 +892,7 @@ impl HostControl for SessionControl { "statusSounds": sounds.mode, "conversationStatus": sounds.status, "sessionArguments": &self.session_arguments[1..], + "audioSetup": berd_call::macos_audio_route::default_audio_setup(), })) } @@ -967,7 +969,16 @@ enum SavedPreference { StatusSounds(berd_call::StatusSoundMode), } +struct ActiveDetectionProfile { + persisted: Option, + setup: berd_call::macos_audio_route::AudioSetup, + path: std::path::PathBuf, + next_check: Instant, + pending: Option, +} + struct SessionActor { + detection_profile: Option, saved: Option<(std::path::PathBuf, crate::saved_settings::SavedSettings)>, input_speaking: bool, recognition_pending: bool, @@ -1082,6 +1093,7 @@ impl SessionActor { audio_commands, next_id: 2, pending_speak: None, + detection_profile: None, pending_settings: std::collections::HashMap::new(), spokesperson_settings: HashSet::new(), waiting_speaks: VecDeque::new(), @@ -1164,8 +1176,91 @@ impl SessionActor { } } + fn restore_detection_profile(&mut self, arguments: &[String]) -> Result<(), String> { + let detection = self + .session + .lock() + .map_err(|_| "session settings lock failed")? + .detection; + let Some(snapshot) = detection else { + return Ok(()); + }; + let Some(setup) = berd_call::macos_audio_route::default_audio_setup() else { + return Ok(()); + }; + let path = crate::detection_profiles::path()?; + let persisted = crate::detection_profiles::load(&path, &setup)?; + let settings = + crate::detection_profiles::settings_for_start(persisted, snapshot.settings, arguments)?; + let id = self.next_id(); + send_request( + &self.writer, + &SessionRequest::SetDetectionSettings { id, settings }, + )?; + self.detection_profile = Some(ActiveDetectionProfile { + persisted, + setup, + path, + next_check: Instant::now() + Duration::from_secs(1), + pending: Some(id), + }); + Ok(()) + } + + fn poll_detection_profile(&mut self) -> Result<(), String> { + let Some(profile) = self.detection_profile.as_mut() else { + return Ok(()); + }; + if profile.pending.is_some() || Instant::now() < profile.next_check { + return Ok(()); + } + profile.next_check = Instant::now() + Duration::from_secs(1); + let Some(setup) = berd_call::macos_audio_route::default_audio_setup() else { + return Ok(()); + }; + self.switch_detection_profile(setup) + } + + fn switch_detection_profile( + &mut self, + setup: berd_call::macos_audio_route::AudioSetup, + ) -> Result<(), String> { + let Some(profile) = self.detection_profile.as_mut() else { + return Ok(()); + }; + if profile.pending.is_some() { + return Ok(()); + } + if setup.input_uid == profile.setup.input_uid + && setup.output_uid == profile.setup.output_uid + { + return Ok(()); + } + let settings = match crate::detection_profiles::load(&profile.path, &setup) { + Ok(settings) => settings, + Err(error) => { + eprintln!("berd-call: could not restore speech profile: {error}"); + return Ok(()); + } + }; + profile.persisted = settings; + let settings = settings.unwrap_or_default(); + profile.setup = setup; + let id = self.next_id(); + send_request( + &self.writer, + &SessionRequest::SetDetectionSettings { id, settings }, + )?; + self.detection_profile + .as_mut() + .expect("profile exists") + .pending = Some(id); + Ok(()) + } + fn poll(&mut self) -> Result<(), String> { if !self.stopping { + self.poll_detection_profile()?; if let Some(muted) = self .system_input_mute .as_mut() @@ -1470,6 +1565,45 @@ impl SessionActor { fn handle_event(&mut self, event: SessionMessage) -> Result<(), String> { match event { + SessionMessage::DetectionSettingsResult { + id, + snapshot, + message, + } => { + if let Some(profile) = self.detection_profile.as_mut() { + if profile.pending.is_some() && profile.pending != id { + return Ok(()); + } + if profile.pending == id { + profile.pending = None; + } + if let Some(snapshot) = snapshot { + // A device switch observed before its ordered acknowledgement + // cannot save the old setup's learning under the new pair. + if berd_call::macos_audio_route::default_audio_setup().is_some_and( + |setup| { + setup.input_uid == profile.setup.input_uid + && setup.output_uid == profile.setup.output_uid + }, + ) { + match crate::detection_profiles::save_if_unchanged(&profile.path, &profile.setup, profile.persisted, snapshot.settings) { + Ok(true) => profile.persisted = Some(snapshot.settings), + Ok(false) => {}, + Err(error) => eprintln!("berd-call: speech detection applied but could not be saved: {error}"), + } + } + } + } + if let Some(snapshot) = snapshot { + self.session + .lock() + .map_err(|_| "session settings lock failed")? + .detection = Some(snapshot); + } + if let Some(message) = message { + eprintln!("berd-call: speech detection update rejected: {message}"); + } + } SessionMessage::InputSpeaking { active } => self.input_speaking = active, SessionMessage::RecognitionPending { active } => self.recognition_pending = active, SessionMessage::State { @@ -4350,6 +4484,7 @@ mod tests { fn test_snapshot() -> VoiceSessionSnapshot { VoiceSessionSnapshot { + detection: None, spokesperson: None, tts: berd_call::TtsConfigurationSnapshot { revision: 1, @@ -4384,6 +4519,91 @@ mod tests { ) } + #[test] + fn device_switch_restores_the_saved_pair_and_ignores_old_learning_before_ack() { + let setup = |name: &str| berd_call::macos_audio_route::AudioSetup { + input_uid: format!("{name}-mic"), + output_uid: format!("{name}-output"), + input_name: name.into(), + output_name: name.into(), + }; + let first = setup("first"); + let second = setup("second"); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("profiles.json"); + let mut first_settings = berd_call::detection::DetectionSettings::default(); + first_settings.listening.value = 0.71; + let mut second_settings = first_settings; + second_settings.interruption.value = 0.84; + crate::detection_profiles::save(&path, &first, first_settings).unwrap(); + crate::detection_profiles::save(&path, &second, second_settings).unwrap(); + let mut child = Command::new("/bin/cat") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let writer = Arc::new(Mutex::new(child.stdin.take().unwrap())); + let (_event_tx, events) = mpsc::sync_channel(1); + let (_command_tx, commands) = mpsc::sync_channel(1); + let (audio, _audio_rx) = mpsc::sync_channel(1); + let mut actor = test_actor(writer, events, commands, audio); + actor.detection_profile = Some(ActiveDetectionProfile { + setup: first.clone(), + path, + persisted: Some(first_settings), + next_check: Instant::now(), + pending: None, + }); + actor.switch_detection_profile(second.clone()).unwrap(); + let id = actor.detection_profile.as_ref().unwrap().pending; + actor + .handle_event(SessionMessage::DetectionSettingsResult { + id: None, + snapshot: Some( + berd_call::detection::DetectionTuner::new(first_settings).snapshot(), + ), + message: None, + }) + .unwrap(); + assert!(actor.session.lock().unwrap().detection.is_none()); + actor + .handle_event(SessionMessage::DetectionSettingsResult { + id, + snapshot: Some( + berd_call::detection::DetectionTuner::new(second_settings).snapshot(), + ), + message: None, + }) + .unwrap(); + assert_eq!( + actor.session.lock().unwrap().detection.unwrap().settings, + second_settings + ); + actor.switch_detection_profile(first).unwrap(); + assert_eq!( + actor.detection_profile.as_ref().unwrap().persisted, + Some(first_settings) + ); + drop(actor); + let output = child.wait_with_output().unwrap().stdout; + let mut position = 0; + let mut settings = Vec::new(); + while position < output.len() { + let length = + u32::from_le_bytes(output[position + 4..position + 8].try_into().unwrap()) as usize; + let request: SessionRequest = + serde_json::from_slice(&output[position + 8..position + 8 + length]).unwrap(); + if let SessionRequest::SetDetectionSettings { + settings: applied, .. + } = request + { + settings.push(applied); + } + position += 8 + length; + } + assert_eq!(settings, [second_settings, first_settings]); + } + #[test] fn polling_waits_for_finalized_input_and_releases_waiters_on_shutdown() { let mut child = Command::new("/bin/cat") diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index e0b4c51dd..b40b47c6f 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -162,6 +162,7 @@ pub struct VoiceInputControls { } struct ControlState { + detection: Mutex>, transition: Mutex, muted: AtomicBool, mute_epoch: AtomicU64, @@ -291,6 +292,7 @@ impl Default for VoiceInputControls { fn default() -> Self { Self { shared: Arc::new(ControlState { + detection: Mutex::new(None), transition: Mutex::new(ControlTransitionState::default()), muted: AtomicBool::new(false), mute_epoch: AtomicU64::new(0), @@ -302,6 +304,73 @@ impl Default for VoiceInputControls { } impl VoiceInputControls { + pub fn set_detection( + &self, + settings: crate::detection::DetectionSettings, + ) -> Result<(), String> { + let settings = settings.validate()?; + let mut detection = self + .shared + .detection + .lock() + .map_err(|_| "speech detection lock failed")?; + self.reset(); + *detection = Some(crate::detection::DetectionTuner::new(settings)); + Ok(()) + } + + pub fn detection(&self) -> Option { + self.shared + .detection + .lock() + .ok()? + .as_ref() + .map(|tuner| tuner.snapshot()) + } + + #[cfg(target_os = "macos")] + fn observe_detection(&self, detected: bool, epoch: u64) { + self.observe_detection_at(detected, epoch, Instant::now()); + } + + #[cfg(target_os = "macos")] + fn observe_detection_at(&self, detected: bool, epoch: u64, now: Instant) { + let mode = if self.shared.assistant_speaking.load(Ordering::Acquire) { + crate::detection::DetectionMode::Interruption + } else { + crate::detection::DetectionMode::Listening + }; + if let Ok(mut detection) = self.shared.detection.lock() { + if self.mute_epoch() != epoch { + return; + } + if let Some(tuner) = detection.as_mut() { + tuner.observe(mode, detected, now); + } + } + } + + #[cfg(target_os = "macos")] + fn detection_result(&self, recognized: bool, epoch: u64) { + self.detection_result_at(recognized, epoch, Instant::now()); + } + + #[cfg(target_os = "macos")] + fn detection_result_at(&self, recognized: bool, epoch: u64, now: Instant) { + if let Ok(mut detection) = self.shared.detection.lock() { + if self.mute_epoch() != epoch { + return; + } + if let Some(tuner) = detection.as_mut() { + if recognized { + tuner.final_transcript(now); + } else { + tuner.no_result(now); + } + } + } + } + pub fn is_muted(&self) -> bool { self.shared.muted.load(Ordering::Acquire) } @@ -414,6 +483,13 @@ impl VoiceInputControls { } fn vad_threshold(&self, speech_threshold: f32) -> f32 { + if let Some(snapshot) = self.detection() { + return if self.shared.assistant_speaking.load(Ordering::Acquire) { + snapshot.settings.interruption.value + } else { + snapshot.settings.listening.value + }; + } if self.shared.assistant_speaking.load(Ordering::Acquire) { f32::from_bits(self.shared.assistant_vad_threshold.load(Ordering::Acquire)) } else { @@ -1391,7 +1467,7 @@ fn new_mac_recognizer( } #[cfg(target_os = "macos")] -fn forward_mac_events( +fn forward_mac_events_with_detection( recognition_events: &mut tokio_mpsc::UnboundedReceiver< crate::mac_speech::MacSpeechRecognitionEvent, >, @@ -1399,13 +1475,19 @@ fn forward_mac_events( pending: &mut PendingRecognitions, settle_deadline: &mut Option, storage_deadline: Option, + controls: Option<(&VoiceInputControls, u64)>, ) -> Result<(), ()> { while let Ok(event) = recognition_events.try_recv() { match event { crate::mac_speech::MacSpeechRecognitionEvent::Final(text) => { let text = text.trim().to_string(); if !text.is_empty() { + if let Some((controls, epoch)) = controls { + controls.detection_result(true, epoch); + } send_final(events, text, storage_deadline)?; + } else if let Some((controls, epoch)) = controls { + controls.detection_result(false, epoch); } pending.resolve(events)?; if pending.count == 0 { @@ -1438,6 +1520,26 @@ fn forward_mac_events( Ok(()) } +#[cfg(all(test, target_os = "macos"))] +fn forward_mac_events( + recognition_events: &mut tokio_mpsc::UnboundedReceiver< + crate::mac_speech::MacSpeechRecognitionEvent, + >, + events: &tokio_mpsc::Sender, + pending: &mut PendingRecognitions, + settle_deadline: &mut Option, + storage_deadline: Option, +) -> Result<(), ()> { + forward_mac_events_with_detection( + recognition_events, + events, + pending, + settle_deadline, + storage_deadline, + None, + ) +} + #[cfg(target_os = "macos")] fn begin_mac_turn( pending: &mut PendingRecognitions, @@ -1504,7 +1606,7 @@ fn mac_speech_worker( let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); let mut input_48k = Vec::new(); let mut leftover_16k = Vec::new(); - let mut silence_frames = 0; + let mut gate = crate::detection::SpeechGate::default(); let mut in_speech = false; let mut observed_epoch = controls.mute_epoch(); let mut pending = PendingRecognitions::new(); @@ -1515,18 +1617,20 @@ fn mac_speech_worker( } loop { - if forward_mac_events( + if forward_mac_events_with_detection( &mut recognition_events, &events, &mut pending, &mut settle_deadline, None, + Some((&controls, observed_epoch)), ) .is_err() { return; } if mac_settle_expired(&pending, settle_deadline, Instant::now()) { + controls.detection_result(false, observed_epoch); // SpeechTranscriber has no per-utterance no-result callback. End // this recognizer generation at the bounded idle deadline so its // suppressed late callbacks cannot resolve a later generation. @@ -1535,7 +1639,7 @@ fn mac_speech_worker( settle_deadline = None; input_48k.clear(); leftover_16k.clear(); - silence_frames = 0; + gate = crate::detection::SpeechGate::default(); vad = earshot::Detector::new(earshot::DefaultPredictor::new()); match new_mac_recognizer(&hints) { Ok((next, next_events)) => { @@ -1558,9 +1662,14 @@ fn mac_speech_worker( effective_mute_epoch(&controls, &shutdown, &shutdown_mute_epoch); if current_epoch != observed_epoch { observed_epoch = current_epoch; + if let Ok(mut detection) = controls.shared.detection.lock() { + if let Some(tuner) = detection.as_mut() { + tuner.reset_evidence(); + } + } input_48k.clear(); leftover_16k.clear(); - silence_frames = 0; + gate = crate::detection::SpeechGate::default(); if std::mem::take(&mut in_speech) { let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); } @@ -1606,32 +1715,34 @@ fn mac_speech_worker( while leftover_16k.len() >= VAD_FRAME_SAMPLES { let frame: Vec = leftover_16k.drain(..VAD_FRAME_SAMPLES).collect(); let clamped = clamp_vad_frame(&frame); - if vad.predict_f32(&clamped) > controls.vad_threshold(speech_vad_threshold) { - silence_frames = 0; - if !in_speech { - in_speech = true; + let detected = + vad.predict_f32(&clamped) > controls.vad_threshold(speech_vad_threshold); + let timing = controls + .detection() + .map(|snapshot| snapshot.settings.timing) + .unwrap_or_default(); + let active = gate.observe(detected, timing); + controls.observe_detection(active && detected, observed_epoch); + if active != in_speech { + in_speech = active; + if active { if begin_mac_turn(&mut pending, &events, &mut settle_deadline).is_err() { return; } - let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(true)); - } - } else if in_speech { - silence_frames += 1; - if silence_frames >= SILENCE_FLUSH_FRAMES { - silence_frames = 0; - in_speech = false; + } else { end_mac_turn(&pending, &mut settle_deadline, Instant::now()); - let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); } + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(active)); } } } - if forward_mac_events( + if forward_mac_events_with_detection( &mut recognition_events, &events, &mut pending, &mut settle_deadline, None, + Some((&controls, observed_epoch)), ) .is_err() { @@ -1653,12 +1764,13 @@ fn mac_speech_worker( return; } let deadline = Instant::now() + FINAL_STORAGE_TIMEOUT; - let _ = forward_mac_events( + let _ = forward_mac_events_with_detection( &mut recognition_events, &events, &mut pending, &mut settle_deadline, Some(deadline), + Some((&controls, observed_epoch)), ); let _ = pending.reset(&events); } @@ -3263,6 +3375,96 @@ mod tests { assert!(!finished.discard_on_shutdown.load(Ordering::Acquire)); } + #[cfg(target_os = "macos")] + #[test] + fn runtime_feedback_changes_only_the_active_threshold_and_applies_without_restart() { + let controls = VoiceInputControls::default(); + let mut settings = crate::detection::DetectionSettings::default(); + settings.listening.dynamic = true; + settings.interruption.dynamic = true; + controls.set_detection(settings).unwrap(); + let epoch = controls.mute_epoch(); + let now = Instant::now(); + for n in 0..1 { + let time = now + Duration::from_secs(n * 40); + controls.observe_detection_at(true, epoch, time); + controls.detection_result_at(false, epoch, time + Duration::from_secs(6)); + } + assert_eq!(controls.vad_threshold(0.5), 0.51); + let playback = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + for n in 0..1 { + let time = now + Duration::from_secs(120 + n * 40); + controls.observe_detection_at(false, epoch, time); + controls.detection_result_at(true, epoch, time + Duration::from_secs(6)); + } + assert_eq!(controls.vad_threshold(0.5), 0.64); + assert_eq!(controls.detection().unwrap().settings.listening.value, 0.51); + drop(playback); + assert_eq!(controls.vad_threshold(0.5), 0.51); + } + + #[cfg(target_os = "macos")] + #[test] + fn stale_input_generation_cannot_train_a_new_profile() { + let controls = VoiceInputControls::default(); + let old_epoch = controls.mute_epoch(); + let mut settings = crate::detection::DetectionSettings::default(); + settings.listening.dynamic = true; + controls.set_detection(settings).unwrap(); + for _ in 0..10 { + controls.observe_detection(true, old_epoch); + controls.detection_result(false, old_epoch); + } + let snapshot = controls.detection().unwrap(); + assert_eq!(snapshot.settings, settings); + assert_eq!(snapshot.no_result, 0); + assert_eq!(snapshot.adjustments, 0); + } + + #[test] + fn applied_detection_settings_survive_playback_guards_and_reject_invalid_updates() { + let controls = VoiceInputControls::default(); + let settings = crate::detection::DetectionSettings { + timing: Default::default(), + listening: crate::detection::Threshold { + value: 0.81, + dynamic: true, + }, + interruption: crate::detection::Threshold { + value: 0.72, + dynamic: false, + }, + }; + controls.set_detection(settings).unwrap(); + assert_eq!(controls.vad_threshold(0.5), 0.81); + let playback = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + assert_eq!(controls.vad_threshold(0.5), 0.72); + drop(playback); + assert_eq!(controls.vad_threshold(0.5), 0.81); + let mut invalid = settings; + invalid.listening.value = f32::NAN; + assert!(controls.set_detection(invalid).is_err()); + assert_eq!(controls.detection().unwrap().settings, settings); + } + + #[test] + fn listening_threshold_applies_before_and_after_independent_playback_threshold() { + let controls = VoiceInputControls::default(); + let listening = 0.8; + let probability = 0.7; + assert!(probability <= controls.vad_threshold(listening)); + let playback = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + assert!(probability > controls.vad_threshold(listening)); + drop(playback); + assert!(probability <= controls.vad_threshold(listening)); + } + #[test] fn silence_does_not_cross_the_default_earshot_threshold() { let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); diff --git a/src-tauri/crates/berd-call/src/lib.rs b/src-tauri/crates/berd-call/src/lib.rs index f3686bfe7..ecdc19644 100644 --- a/src-tauri/crates/berd-call/src/lib.rs +++ b/src-tauri/crates/berd-call/src/lib.rs @@ -8,6 +8,7 @@ mod audio_output; pub mod benchmark; pub mod causal_inbox; mod configured_tts; +pub mod detection; pub mod endpoint_url; pub mod expert_spokesperson; pub mod input; diff --git a/src-tauri/crates/berd-call/src/macos_audio_route.rs b/src-tauri/crates/berd-call/src/macos_audio_route.rs index ca890e116..ffd8efe45 100644 --- a/src-tauri/crates/berd-call/src/macos_audio_route.rs +++ b/src-tauri/crates/berd-call/src/macos_audio_route.rs @@ -165,3 +165,51 @@ mod tests { )); } } + +/// Stable default-device identities used to isolate local speech calibration. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +pub struct AudioSetup { + pub input_uid: String, + pub output_uid: String, + pub input_name: String, + pub output_name: String, +} + +pub fn default_audio_setup() -> Option { + fn uid(device: AudioObjectID) -> Option { + use objc2_core_audio::kAudioDevicePropertyDeviceUID; + use objc2_core_foundation::{CFRetained, CFString}; + let address = AudioObjectPropertyAddress { + mSelector: kAudioDevicePropertyDeviceUID, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + let mut value: *const CFString = null(); + let mut size = mem::size_of_val(&value) as u32; + // SAFETY: CoreAudio writes one retained CFString pointer into a correctly + // sized live buffer. CFRetained owns and releases the returned reference. + let status = unsafe { + AudioObjectGetPropertyData( + device, + NonNull::from(&address), + 0, + null(), + NonNull::from(&mut size), + NonNull::from(&mut value).cast(), + ) + }; + if status != kAudioHardwareNoError { + return None; + } + let pointer = NonNull::new(value.cast_mut())?; + Some(unsafe { CFRetained::from_raw(pointer) }.to_string()) + } + let input = get_default_device_id(true)?; + let output = get_default_device_id(false)?; + Some(AudioSetup { + input_uid: uid(input)?, + output_uid: uid(output)?, + input_name: get_device_name(input).ok()?, + output_name: get_device_name(output).ok()?, + }) +} diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 7d4872a73..fb0ab5404 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -66,6 +66,8 @@ use serde::Serialize; mod cli_help; #[cfg(target_os = "macos")] mod codex; +#[cfg(target_os = "macos")] +mod detection_profiles; mod host_control; #[cfg(unix)] mod host_lifetime; @@ -208,6 +210,10 @@ struct SessionConfig { /// The Spokesperson's own voice and speed when replies have theirs. spokesperson_voice: Option, spokesperson_rate: Option, + interruption_threshold: f32, + listening_threshold: f32, + dynamic_listening: bool, + dynamic_interruption: bool, /// The Spokesperson's session options beyond its voice and speed. spokesperson_options: Option, @@ -544,6 +550,32 @@ fn main() { None => {} } match args.get(1).map(String::as_str) { + #[cfg(target_os = "macos")] + Some("detection-profile") => { + let result = (|| -> Result { + let setup = berd_call::macos_audio_route::default_audio_setup() + .ok_or("audio setup is unavailable")?; + let path = detection_profiles::path()?; + if args.len() > 2 { + if args.len() != 4 || args[2] != "--settings" { + return Err("usage: berd-call detection-profile [--settings JSON]".into()); + } + let settings: berd_call::detection::DetectionSettings = + serde_json::from_str(&args[3]).map_err(|error| error.to_string())?; + detection_profiles::save(&path, &setup, settings)?; + } + Ok( + serde_json::json!({"audioSetup": setup, "settings": detection_profiles::load(&path, &setup)?.unwrap_or_default()}), + ) + })(); + match result { + Ok(value) => println!("{value}"), + Err(error) => { + eprintln!("berd-call detection-profile failed: {error}"); + std::process::exit(1); + } + } + } Some("start") => { let options = parse_or_exit(parse_saved_start_args(&args), &args); #[cfg(target_os = "macos")] @@ -2213,6 +2245,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let mut next_tts_update_attempt = 1_u64; let mut input_runtime: Option = None; let mut input_events: Option> = None; + let mut last_detection = None; let mut input_controls: Option = None; let mut next_input_token = 1_u64; let mut pending_control = None; @@ -2271,6 +2304,22 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String &mut tts_update, &mut writer, )?; + if let Some(snapshot) = input_controls + .as_ref() + .and_then(|controls| controls.detection()) + { + if last_detection != Some(snapshot) { + write_message( + &mut writer, + &SessionMessage::DetectionSettingsResult { + id: None, + snapshot: Some(snapshot), + message: None, + }, + )?; + last_detection = Some(snapshot); + } + } reevaluate_held( &mut held, &mut core, @@ -2438,6 +2487,17 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String &config.stt, config.endpoints.stt.as_deref(), &config.stt_hints, + berd_call::detection::DetectionSettings { + timing: Default::default(), + listening: berd_call::detection::Threshold { + value: config.listening_threshold, + dynamic: config.dynamic_listening, + }, + interruption: berd_call::detection::Threshold { + value: config.interruption_threshold, + dynamic: config.dynamic_interruption, + }, + }, ) { Ok(runtime) => runtime, Err(message) => { @@ -2470,10 +2530,14 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String status_sound_runtime.set_output_device(status_sound_output_device); let input_policy = InputDuringTtsSlot::new(input_during_tts); let session = VoiceSessionSnapshot { + detection: input_controls + .as_ref() + .and_then(|controls| controls.detection()), tts: slot.snapshot()?, input_during_tts: input_policy.snapshot()?, spokesperson: None, }; + last_detection = session.detection; tts_slot = Some(slot); input_during_tts_slot = Some(input_policy); write_message( @@ -2574,6 +2638,36 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String }); } } + Input::Request(SessionRequest::SetDetectionSettings { id, settings }) => { + let result = if (settings.listening.dynamic || settings.interruption.dynamic) + && !matches!(config.stt, SttBackendConfig::Macos) + { + Err( + "dynamic speech detection currently requires macOS speech recognition" + .to_string(), + ) + } else { + input_controls + .as_ref() + .ok_or_else(|| "speech input is not ready".to_string()) + .and_then(|controls| controls.set_detection(settings)) + }; + let snapshot = result + .as_ref() + .ok() + .and_then(|_| input_controls.as_ref()?.detection()); + if snapshot.is_some() { + last_detection = snapshot; + } + write_message( + &mut writer, + &SessionMessage::DetectionSettingsResult { + id: Some(id), + snapshot, + message: result.err(), + }, + )?; + } Input::Request(SessionRequest::SetInputDuringTts { id, expected_revision, @@ -2659,7 +2753,12 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String && current.speech_id == speech_id && current.active.is_none() }) { - acknowledge_output_ready(current, input_controls.as_ref(), &mut writer)?; + acknowledge_output_ready( + current, + input_controls.as_ref(), + config.interruption_threshold, + &mut writer, + )?; let playback_active = Arc::new(AtomicBool::new(true)); let output = Arc::new(RemotePcmAudioOutput::new( speech_id, @@ -4778,6 +4877,7 @@ fn run_expert_spokesperson_session( }; let input_policy = InputDuringTtsSlot::new(input_during_tts); let snapshot = VoiceSessionSnapshot { + detection: None, tts: match &reply_tts { Some(slot) => slot.snapshot()?, None => tts.clone(), @@ -5282,6 +5382,18 @@ fn run_expert_spokesperson_session( &mut writer, )?; } + Input::Request(SessionRequest::SetDetectionSettings { id, .. }) => { + write_message( + &mut writer, + &SessionMessage::DetectionSettingsResult { + id: Some(id), + snapshot: None, + message: Some( + "local speech detection controls require a conventional call".into(), + ), + }, + )?; + } Input::Request(SessionRequest::SetInputDuringTts { id, expected_revision, @@ -6173,12 +6285,13 @@ fn expert_spoken_through_utf8(response: &LiveResponse) -> u64 { fn acknowledge_output_ready( current: &mut ActivePlayback, input_controls: Option<&VoiceInputControls>, + interruption_threshold: f32, writer: &mut impl Write, ) -> Result<(), String> { current.assistant_activity = input_controls.map(|controls| { controls - .begin_assistant_activity(0.65, current.input_during_tts.policy) - .expect("balanced assistant threshold is valid") + .begin_assistant_activity(interruption_threshold, current.input_during_tts.policy) + .expect("validated interruption threshold is valid") }); write_message( writer, @@ -6284,6 +6397,10 @@ fn parse_args(args: &[String]) -> Result { let mut expert_tts = false; let mut spokesperson_voice = None; let mut spokesperson_rate = None; + let mut interruption_threshold = 0.65; + let mut listening_threshold = 0.5; + let mut dynamic_listening = false; + let mut dynamic_interruption = false; let mut spokesperson_options = None; let mut index = 2; while index < args.len() { @@ -6300,6 +6417,29 @@ fn parse_args(args: &[String]) -> Result { .get(index + 1) .ok_or_else(|| format!("{flag} requires a value"))?; match flag { + "--interruption-threshold" | "--listening-threshold" => { + let threshold = value + .parse::() + .map_err(|_| format!("{flag} must be a number"))?; + if !threshold.is_finite() || !(0.0..=1.0).contains(&threshold) { + return Err(format!("{flag} must be between 0 and 1").into()); + } + if flag == "--listening-threshold" { + listening_threshold = threshold; + } else { + interruption_threshold = threshold; + } + } + "--dynamic-listening" | "--dynamic-interruption" => { + let enabled = value + .parse::() + .map_err(|_| format!("{flag} must be true or false"))?; + if flag == "--dynamic-listening" { + dynamic_listening = enabled; + } else { + dynamic_interruption = enabled; + } + } "--tts-backend" => backend = value, "--voice" => voice = Some(value.clone()), "--language" => language = Some(value.clone()), @@ -6397,6 +6537,10 @@ fn parse_args(args: &[String]) -> Result { expert_tts, spokesperson_voice, spokesperson_rate, + interruption_threshold, + listening_threshold, + dynamic_listening, + dynamic_interruption, spokesperson_options, }) } @@ -7017,7 +7161,7 @@ fn create_stt_benchmark_report( &pack, config.runs, config.mode, - || create_input_runtime(&config.stt, None, &[]), + || create_input_runtime(&config.stt, None, &[], Default::default()), ))) } @@ -7358,6 +7502,7 @@ fn create_input_runtime( config: &SttBackendConfig, endpoint_override: Option<&str>, _hints: &[String], + detection: berd_call::detection::DetectionSettings, ) -> Result< ( VoiceInputRuntime, @@ -7420,10 +7565,17 @@ fn create_input_runtime( } } }; + if (detection.listening.dynamic || detection.interruption.dynamic) + && !matches!(config, SttBackendConfig::Macos) + { + return Err("dynamic speech detection currently requires macOS speech recognition".into()); + } + let controls = VoiceInputControls::default(); + controls.set_detection(detection)?; VoiceInputRuntime::start(VoiceInputConfig { engine, - speech_vad_threshold: 0.5, - controls: VoiceInputControls::default(), + speech_vad_threshold: detection.listening.value, + controls, }) .map_err(|error| match config { SttBackendConfig::Macos => format!( @@ -8463,7 +8615,8 @@ fn decode_framed_input(reader: &mut impl Read, header: [u8; FRAME_HEADER_BYTES]) fn validate_request(request: SessionRequest) -> Result { let id = match &request { - SessionRequest::Hello { id, .. } + SessionRequest::SetDetectionSettings { id, .. } + | SessionRequest::Hello { id, .. } | SessionRequest::SetInputMuted { id, .. } | SessionRequest::SetConversationStatus { id, .. } | SessionRequest::SetTtsSettings { id, .. } @@ -8748,6 +8901,71 @@ mod tests { use std::os::unix::net::UnixStream; use std::sync::Mutex; + #[test] + fn dynamic_detection_switches_are_explicit_and_independent() { + let base: Vec = ["berd-call", "session", "--tts-backend", "openai"] + .map(str::to_string) + .to_vec(); + let config = parse_args(&base).unwrap(); + assert!(!config.dynamic_listening && !config.dynamic_interruption); + let mut args = base.clone(); + args.extend(["--dynamic-listening".into(), "true".into()]); + let config = parse_args(&args).unwrap(); + assert!(config.dynamic_listening && !config.dynamic_interruption); + args.extend(["--dynamic-interruption".into(), "true".into()]); + assert!(parse_args(&args).unwrap().dynamic_interruption); + args.extend(["--dynamic-listening".into(), "maybe".into()]); + assert!(parse_args(&args).is_err()); + } + + #[test] + fn listening_threshold_is_independent_and_bounded() { + let base: Vec = ["berd-call", "session", "--tts-backend", "openai"] + .map(str::to_string) + .to_vec(); + assert_eq!(parse_args(&base).unwrap().listening_threshold, 0.5); + for value in ["0", "0.8", "1"] { + let mut args = base.clone(); + args.extend(["--listening-threshold".into(), value.into()]); + let config = parse_args(&args).unwrap(); + assert_eq!(config.interruption_threshold, 0.65); + assert_eq!(config.listening_threshold, value.parse::().unwrap()); + args.extend(["--interruption-threshold".into(), "0.7".into()]); + let config = parse_args(&args).unwrap(); + assert_eq!(config.interruption_threshold, 0.7); + assert_eq!(config.listening_threshold, value.parse::().unwrap()); + } + for value in ["-0.1", "1.1", "NaN", "inf", "quiet"] { + let mut args = base.clone(); + args.extend(["--listening-threshold".into(), value.into()]); + assert!(parse_args(&args).is_err(), "{value}"); + } + } + + #[test] + fn interruption_threshold_is_bounded_and_defaults_to_balanced() { + let base = vec![ + "berd-call".into(), + "session".into(), + "--tts-backend".into(), + "openai".into(), + ]; + assert_eq!(parse_args(&base).unwrap().interruption_threshold, 0.65); + for value in ["0", "0.8", "1"] { + let mut args = base.clone(); + args.extend(["--interruption-threshold".into(), value.into()]); + assert_eq!( + parse_args(&args).unwrap().interruption_threshold, + value.parse::().unwrap() + ); + } + for value in ["-0.1", "1.1", "NaN", "inf", "quiet"] { + let mut args = base.clone(); + args.extend(["--interruption-threshold".into(), value.into()]); + assert!(parse_args(&args).is_err(), "{value}"); + } + } + #[cfg(target_os = "macos")] #[test] fn update_check_only_targets_the_owning_app_bundle() { @@ -10551,6 +10769,7 @@ mod tests { id: 1, protocol: SESSION_PROTOCOL_VERSION, session: VoiceSessionSnapshot { + detection: None, tts: snapshot.clone(), input_during_tts: test_input_policy(), spokesperson: None, @@ -11173,6 +11392,10 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, + listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, spokesperson_options: None, } ); @@ -11188,6 +11411,10 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, + listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, spokesperson_options: None, } ); @@ -11269,6 +11496,10 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, + listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, spokesperson_options: None, } ); @@ -11329,6 +11560,10 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, + listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, spokesperson_options: None, }; let mut realtime = OpenAiSpokespersonConfig { @@ -11615,6 +11850,10 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, + listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, spokesperson_options: None, } ); @@ -11681,6 +11920,10 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, + listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, spokesperson_options: None, } ); @@ -12130,6 +12373,10 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, + listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, spokesperson_options: None, } ); @@ -12527,6 +12774,60 @@ mod tests { ); } + #[test] + fn false_speech_hold_releases_only_after_speaking_and_recognition_settle() { + let mut core = SessionCore::default(); + let mut active = None; + let mut held = None; + let mut output = Vec::new(); + let mut next_token = 1; + for event in [ + VoiceInputEvent::RecognitionPendingChanged(true), + VoiceInputEvent::SpeakingChanged(true), + ] { + handle_voice_input_event(event, &mut core, &mut active, &mut next_token, &mut output) + .unwrap(); + } + process_prepare( + PrepareRequest { + id: 4, + acknowledgement: None, + text: "reply".into(), + }, + None, + &mut core, + &test_tts_slot(), + &test_input_policy_slot(), + &mut active, + &mut held, + &mut output, + ) + .unwrap(); + assert!(held.is_some()); + assert!(active.is_none()); + for pending in [true, false] { + let event = if pending { + VoiceInputEvent::SpeakingChanged(false) + } else { + VoiceInputEvent::RecognitionPendingChanged(false) + }; + handle_voice_input_event(event, &mut core, &mut active, &mut next_token, &mut output) + .unwrap(); + reevaluate_held( + &mut held, + &mut core, + Some(&test_tts_slot()), + Some(&test_input_policy_slot()), + &mut active, + &mut output, + ) + .unwrap(); + assert_eq!(held.is_some(), pending); + assert_eq!(active.is_some(), !pending); + } + assert_eq!(messages(&output).last().unwrap()["type"], "admitted"); + } + #[test] fn held_prepare_waits_for_pending_to_clear_without_a_timeout() { let mut core = SessionCore::default(); @@ -12853,7 +13154,7 @@ mod tests { bytes: Vec::new(), }; - acknowledge_output_ready(&mut current, Some(&controls), &mut writer).unwrap(); + acknowledge_output_ready(&mut current, Some(&controls), 0.8, &mut writer).unwrap(); assert!(current.assistant_activity.is_some()); assert_eq!( @@ -12877,7 +13178,7 @@ mod tests { }; let controls = VoiceInputControls::default(); let mut ignored = Vec::new(); - acknowledge_output_ready(&mut current, Some(&controls), &mut ignored).unwrap(); + acknowledge_output_ready(&mut current, Some(&controls), 0.65, &mut ignored).unwrap(); assert!(controls.is_muted()); let speech_id = current.speech_id; let mut active = Some(current); diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index cf6d578d5..440777087 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -9,6 +9,10 @@ use crate::{ #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] pub enum SessionRequest { + SetDetectionSettings { + id: u64, + settings: crate::detection::DetectionSettings, + }, Hello { id: u64, input_during_tts: InputDuringTtsPolicy, @@ -173,6 +177,8 @@ pub enum OutputReadyOutcome { pub struct VoiceSessionSnapshot { /// The voice that speaks replies: the conventional backend, or the /// Spokesperson's own voice when it reads them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detection: Option, pub tts: TtsConfigurationSnapshot, pub input_during_tts: InputDuringTtsSnapshot, /// The Spokesperson's voice, when replies have their own. @@ -213,6 +219,11 @@ pub enum ExpertTurnOutcome { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum SessionMessage { + DetectionSettingsResult { + id: Option, + snapshot: Option, + message: Option, + }, Ready { id: u64, protocol: u32, @@ -384,7 +395,7 @@ mod tests { #[test] fn host_decodes_realtime_ready_snapshot() { - let message = r#"{"type":"ready","id":1,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-realtime-2.1","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"#; + let message = r#"{"type":"ready","id":1,"protocol":7,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-realtime-2.1","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"#; let decoded = serde_json::from_str::(message).unwrap(); assert!(matches!(decoded, SessionMessage::Ready { id: 1, .. })); } @@ -410,8 +421,9 @@ mod tests { assert_eq!( serde_json::to_string(&SessionMessage::Ready { id: 4, - protocol: 5, + protocol: 7, session: VoiceSessionSnapshot { + detection: None, tts: TtsConfigurationSnapshot { revision: 1, settings: TtsSettings::OpenAi { @@ -435,7 +447,7 @@ mod tests { }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"revision":1,"backend":"openai","model":"gpt-realtime","voice":"cedar","rate":1.0}}}"# + r#"{"type":"ready","id":4,"protocol":7,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"revision":1,"backend":"openai","model":"gpt-realtime","voice":"cedar","rate":1.0}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index a5bb2253c..943f3b478 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -104,6 +104,10 @@ fn validate_arguments(arguments: &[String]) -> Result<(), String> { | "--stt-model-dir" | "--spokesperson-voice" | "--spokesperson-rate" + | "--interruption-threshold" + | "--listening-threshold" + | "--dynamic-listening" + | "--dynamic-interruption" ) }) { @@ -139,14 +143,15 @@ pub(crate) fn persistable_arguments(arguments: &[String]) -> Vec { arguments .chunks_exact(2) .filter(|pair| { - !matches!( - pair[0].as_str(), - "--realtime-url" - | "--stt-url" - | "--tts-url" - | "--spokesperson-options" - | "--stt-hint" - ) + !berd_call::detection::is_detection_flag(&pair[0]) + && !matches!( + pair[0].as_str(), + "--realtime-url" + | "--stt-url" + | "--tts-url" + | "--spokesperson-options" + | "--stt-hint" + ) }) .flatten() .cloned() @@ -235,6 +240,38 @@ pub(crate) fn with_spokesperson(arguments: &[String], on: bool, direct: bool) -> mod tests { use super::*; + #[test] + fn detection_thresholds_survive_reopen_and_override_independently() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let settings = SavedSettings { + arguments: [ + "--listening-threshold", + "0.8", + "--interruption-threshold", + "0.7", + ] + .map(str::to_string) + .to_vec(), + ..SavedSettings::default() + }; + save(&path, &settings).unwrap(); + let restored = load(&path).unwrap(); + assert_eq!(restored.arguments, settings.arguments); + assert_eq!( + merge_arguments( + &restored.arguments, + &["--interruption-threshold".into(), "0.9".into()] + ), + [ + "--listening-threshold", + "0.8", + "--interruption-threshold", + "0.9" + ] + ); + } + #[test] fn auto_and_explicit_input_preferences_survive_reopen() { let directory = tempfile::tempdir().unwrap(); diff --git a/src-tauri/crates/berd-call/src/session_framing.rs b/src-tauri/crates/berd-call/src/session_framing.rs index 5401ab4fa..94fa6aa49 100644 --- a/src-tauri/crates/berd-call/src/session_framing.rs +++ b/src-tauri/crates/berd-call/src/session_framing.rs @@ -1,4 +1,4 @@ -pub(crate) const SESSION_PROTOCOL_VERSION: u32 = 5; +pub(crate) const SESSION_PROTOCOL_VERSION: u32 = 7; pub(crate) const FRAME_MAGIC: [u8; 2] = *b"BV"; pub(crate) const FRAME_MARKER: u8 = 3; pub(crate) const JSON_FRAME_KIND: u8 = 1; diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 3c7cb49fb..59b26c242 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -2950,7 +2950,7 @@ fn siri_session_reaches_ready_without_openai_credentials() { stdin.flush().unwrap(); let ready = receive(); assert_eq!(ready["type"], "ready"); - assert_eq!(ready["protocol"], 5); + assert_eq!(ready["protocol"], 7); assert_eq!(ready["session"]["tts"]["backend"], "siri"); assert_eq!(ready["session"]["tts"]["voice"], voice); assert_eq!(ready["session"]["tts"]["language"], language);