From 11bd8b4bd551552e81b5cce771ace82264944694 Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Wed, 21 Oct 2026 21:35:56 -0400 Subject: [PATCH 01/10] berd-call: configure local interruption detection threshold Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/README.md | 4 ++ src-tauri/crates/berd-call/src/cli_help.rs | 5 +- src-tauri/crates/berd-call/src/main.rs | 60 +++++++++++++++++-- .../crates/berd-call/src/saved_settings.rs | 1 + 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 6ca79f4ab..06ea4a4f1 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -9,6 +9,10 @@ management, synthesis and benchmark tools described below. Desktop Berd keeps its persisted settings and user interface. The standalone command includes a small macOS host for default-device capture, playback, and transcript delivery. +### Local interruption detection + +Conventional calls accept `--interruption-threshold 0..1` at `start` or `session` (default `0.65`). It controls the local speech detector while an agent's audio plays. Higher values require stronger evidence of speech before pausing playback; lower values make interruptions easier. It does not change the idle listening threshold or the transcription provider. Saved call arguments retain the value. Use `settings --restart --interruption-threshold 0.8` to change a running call; this restarts its session. + ## Command-line interface On macOS, install [Berd](https://github.com/block/berd/releases) first. Its diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 487ad4bf1..512b07c28 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -147,12 +147,15 @@ Usage: --model-dir ABSOLUTE_PATH --voice ID Each form also accepts: - [--rate FLOAT] + [--rate FLOAT] [--interruption-threshold 0..1] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] [--stt-hint PHRASE]... [--mode conventional|expert-spokesperson] [--allow-direct-agent-speech] [--spokesperson-voice NAME] [--spokesperson-rate 0.25..1.5] [--realtime-url WS_URL] [--stt-url WS_URL] [--tts-url HTTP_URL] +--interruption-threshold sets local speech detection during agent playback (default 0.65). +Higher values require stronger evidence of speech before pausing playback. + The host owns microphone capture, audio playback, transcript delivery, and agent integration. See PROTOCOL.md for the framed stdin, stdout, and PCM contracts. Endpoint URL flags override only the matching service for this call diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index f2f6501fb..4d64ca3ba 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -206,6 +206,7 @@ struct SessionConfig { /// The Spokesperson's own voice and speed when replies have theirs. spokesperson_voice: Option, spokesperson_rate: Option, + interruption_threshold: f32, } #[derive(Clone, Debug, Default, PartialEq)] @@ -2518,7 +2519,12 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String && current.speech_id == speech_id && current.active.is_none() }) { - acknowledge_output_ready(current, input_controls.as_ref(), &mut writer)?; + acknowledge_output_ready( + current, + input_controls.as_ref(), + config.interruption_threshold, + &mut writer, + )?; let playback_active = Arc::new(AtomicBool::new(true)); let output = Arc::new(RemotePcmAudioOutput::new( speech_id, @@ -5959,12 +5965,13 @@ fn expert_spoken_through_utf8(response: &LiveResponse) -> u64 { fn acknowledge_output_ready( current: &mut ActivePlayback, input_controls: Option<&VoiceInputControls>, + interruption_threshold: f32, writer: &mut impl Write, ) -> Result<(), String> { current.assistant_activity = input_controls.map(|controls| { controls - .begin_assistant_activity(0.65, current.input_during_tts.policy) - .expect("balanced assistant threshold is valid") + .begin_assistant_activity(interruption_threshold, current.input_during_tts.policy) + .expect("validated interruption threshold is valid") }); write_message( writer, @@ -6070,6 +6077,7 @@ fn parse_args(args: &[String]) -> Result { let mut expert_tts = false; let mut spokesperson_voice = None; let mut spokesperson_rate = None; + let mut interruption_threshold = 0.65; let mut index = 2; while index < args.len() { let flag = args[index].as_str(); @@ -6085,6 +6093,16 @@ fn parse_args(args: &[String]) -> Result { .get(index + 1) .ok_or_else(|| format!("{flag} requires a value"))?; match flag { + "--interruption-threshold" => { + interruption_threshold = value + .parse::() + .map_err(|_| "--interruption-threshold must be a number")?; + if !interruption_threshold.is_finite() + || !(0.0..=1.0).contains(&interruption_threshold) + { + return Err("--interruption-threshold must be between 0 and 1".into()); + } + } "--tts-backend" => backend = value, "--voice" => voice = Some(value.clone()), "--language" => language = Some(value.clone()), @@ -6177,6 +6195,7 @@ fn parse_args(args: &[String]) -> Result { expert_tts, spokesperson_voice, spokesperson_rate, + interruption_threshold, }) } @@ -8488,6 +8507,30 @@ mod tests { use std::os::unix::net::UnixStream; use std::sync::Mutex; + #[test] + fn interruption_threshold_is_bounded_and_defaults_to_balanced() { + let base = vec![ + "berd-call".into(), + "session".into(), + "--tts-backend".into(), + "openai".into(), + ]; + assert_eq!(parse_args(&base).unwrap().interruption_threshold, 0.65); + for value in ["0", "0.8", "1"] { + let mut args = base.clone(); + args.extend(["--interruption-threshold".into(), value.into()]); + assert_eq!( + parse_args(&args).unwrap().interruption_threshold, + value.parse::().unwrap() + ); + } + for value in ["-0.1", "1.1", "NaN", "inf", "quiet"] { + let mut args = base.clone(); + args.extend(["--interruption-threshold".into(), value.into()]); + assert!(parse_args(&args).is_err(), "{value}"); + } + } + #[cfg(target_os = "macos")] #[test] fn update_check_only_targets_the_owning_app_bundle() { @@ -10772,6 +10815,7 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, } ); @@ -10786,6 +10830,7 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, } ); } @@ -10866,6 +10911,7 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, } ); assert!(parse_args(&args(&[ @@ -10925,6 +10971,7 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, }; let mut realtime = OpenAiSpokespersonConfig { endpoint: "ws://localhost".into(), @@ -11148,6 +11195,7 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, } ); assert!(parse_args(&args(&[ @@ -11213,6 +11261,7 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, } ); assert!(parse_args(&args(&[ @@ -11661,6 +11710,7 @@ mod tests { expert_tts: false, spokesperson_voice: None, spokesperson_rate: None, + interruption_threshold: 0.65, } ); } @@ -12218,7 +12268,7 @@ mod tests { bytes: Vec::new(), }; - acknowledge_output_ready(&mut current, Some(&controls), &mut writer).unwrap(); + acknowledge_output_ready(&mut current, Some(&controls), 0.8, &mut writer).unwrap(); assert!(current.assistant_activity.is_some()); assert_eq!( @@ -12242,7 +12292,7 @@ mod tests { }; let controls = VoiceInputControls::default(); let mut ignored = Vec::new(); - acknowledge_output_ready(&mut current, Some(&controls), &mut ignored).unwrap(); + acknowledge_output_ready(&mut current, Some(&controls), 0.65, &mut ignored).unwrap(); assert!(controls.is_muted()); let speech_id = current.speech_id; let mut active = Some(current); diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index aa21d17d0..706878074 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -104,6 +104,7 @@ fn validate_arguments(arguments: &[String]) -> Result<(), String> { | "--stt-model-dir" | "--spokesperson-voice" | "--spokesperson-rate" + | "--interruption-threshold" ) }) { From 14db28b4869b73517add5462b66ff6a8b83a076f Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Wed, 21 Oct 2026 21:38:34 -0400 Subject: [PATCH 02/10] docs: give complete interruption threshold restart example Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 06ea4a4f1..8cbe95777 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -11,7 +11,7 @@ small macOS host for default-device capture, playback, and transcript delivery. ### Local interruption detection -Conventional calls accept `--interruption-threshold 0..1` at `start` or `session` (default `0.65`). It controls the local speech detector while an agent's audio plays. Higher values require stronger evidence of speech before pausing playback; lower values make interruptions easier. It does not change the idle listening threshold or the transcription provider. Saved call arguments retain the value. Use `settings --restart --interruption-threshold 0.8` to change a running call; this restarts its session. +Conventional calls accept `--interruption-threshold 0..1` at `start` or `session` (default `0.65`). It controls the local speech detector while an agent's audio plays. Higher values require stronger evidence of speech before pausing playback; lower values make interruptions easier. It does not change the idle listening threshold or the transcription provider. Saved call arguments retain the value. To change a running call, supply its complete session configuration, for example `settings --restart --voice Aaron --language en-US --interruption-threshold 0.8`. Restart replaces the session options; include the existing backend, model, rate and other choices you want to retain. ## Command-line interface From bf93a2842504785cdbc0911b0876e96cbbc94543 Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Thu, 22 Oct 2026 06:51:31 -0400 Subject: [PATCH 03/10] berd-call: configure independent listening speech detection Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/README.md | 8 +- src-tauri/crates/berd-call/src/cli_help.rs | 3 +- src-tauri/crates/berd-call/src/input.rs | 14 +++ src-tauri/crates/berd-call/src/main.rs | 111 ++++++++++++++++-- .../crates/berd-call/src/saved_settings.rs | 33 ++++++ 5 files changed, 157 insertions(+), 12 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 8cbe95777..684090d9c 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -9,9 +9,13 @@ management, synthesis and benchmark tools described below. Desktop Berd keeps its persisted settings and user interface. The standalone command includes a small macOS host for default-device capture, playback, and transcript delivery. -### Local interruption detection +### Local speech detection -Conventional calls accept `--interruption-threshold 0..1` at `start` or `session` (default `0.65`). It controls the local speech detector while an agent's audio plays. Higher values require stronger evidence of speech before pausing playback; lower values make interruptions easier. It does not change the idle listening threshold or the transcription provider. Saved call arguments retain the value. To change a running call, supply its complete session configuration, for example `settings --restart --voice Aaron --language en-US --interruption-threshold 0.8`. Restart replaces the session options; include the existing backend, model, rate and other choices you want to retain. +Conventional calls accept independent `--listening-threshold 0..1` (default `0.5`) and `--interruption-threshold 0..1` (default `0.65`) at `start` or `session`. Higher values require stronger evidence of speech. Listening controls detection before agent playback, including the gate that holds replies while input is speaking or awaiting recognition. Interruption controls detection during agent playback. These local thresholds do not change provider endpointing or Spokesperson server VAD. + +On macOS, a local speech detection also marks recognition pending. After local speech ends, admission continues waiting for recognition to settle, with a five-second no-result fallback. Raising the listening threshold can reduce false starts in background noise; it does not shorten that fallback or prove a particular microphone's false detections are resolved. + +Saved call arguments retain both values independently. To change a running call, supply its complete session configuration, for example `settings --restart --voice Aaron --language en-US --listening-threshold 0.8 --interruption-threshold 0.7`. Restart replaces the session options; include the existing backend, model, rate and other choices you want to retain. ## Command-line interface diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 512b07c28..23e6d768e 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -147,12 +147,13 @@ Usage: --model-dir ABSOLUTE_PATH --voice ID Each form also accepts: - [--rate FLOAT] [--interruption-threshold 0..1] + [--rate FLOAT] [--listening-threshold 0..1] [--interruption-threshold 0..1] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] [--stt-hint PHRASE]... [--mode conventional|expert-spokesperson] [--allow-direct-agent-speech] [--spokesperson-voice NAME] [--spokesperson-rate 0.25..1.5] [--realtime-url WS_URL] [--stt-url WS_URL] [--tts-url HTTP_URL] +--listening-threshold sets local speech detection before agent playback (default 0.5). --interruption-threshold sets local speech detection during agent playback (default 0.65). Higher values require stronger evidence of speech before pausing playback. diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index ed742e18e..d1e48dadd 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -3051,6 +3051,20 @@ mod tests { assert!(!finished.discard_on_shutdown.load(Ordering::Acquire)); } + #[test] + fn listening_threshold_applies_before_and_after_independent_playback_threshold() { + let controls = VoiceInputControls::default(); + let listening = 0.8; + let probability = 0.7; + assert!(probability <= controls.vad_threshold(listening)); + let playback = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + assert!(probability > controls.vad_threshold(listening)); + drop(playback); + assert!(probability <= controls.vad_threshold(listening)); + } + #[test] fn silence_does_not_cross_the_default_earshot_threshold() { let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 4d64ca3ba..0c64188c4 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -207,6 +207,7 @@ struct SessionConfig { spokesperson_voice: Option, spokesperson_rate: Option, interruption_threshold: f32, + listening_threshold: f32, } #[derive(Clone, Debug, Default, PartialEq)] @@ -2299,6 +2300,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String &config.stt, config.endpoints.stt.as_deref(), &config.stt_hints, + config.listening_threshold, ) { Ok(runtime) => runtime, Err(message) => { @@ -6078,6 +6080,7 @@ fn parse_args(args: &[String]) -> Result { let mut spokesperson_voice = None; let mut spokesperson_rate = None; let mut interruption_threshold = 0.65; + let mut listening_threshold = 0.5; let mut index = 2; while index < args.len() { let flag = args[index].as_str(); @@ -6093,14 +6096,17 @@ fn parse_args(args: &[String]) -> Result { .get(index + 1) .ok_or_else(|| format!("{flag} requires a value"))?; match flag { - "--interruption-threshold" => { - interruption_threshold = value + "--interruption-threshold" | "--listening-threshold" => { + let threshold = value .parse::() - .map_err(|_| "--interruption-threshold must be a number")?; - if !interruption_threshold.is_finite() - || !(0.0..=1.0).contains(&interruption_threshold) - { - return Err("--interruption-threshold must be between 0 and 1".into()); + .map_err(|_| format!("{flag} must be a number"))?; + if !threshold.is_finite() || !(0.0..=1.0).contains(&threshold) { + return Err(format!("{flag} must be between 0 and 1").into()); + } + if flag == "--listening-threshold" { + listening_threshold = threshold; + } else { + interruption_threshold = threshold; } } "--tts-backend" => backend = value, @@ -6196,6 +6202,7 @@ fn parse_args(args: &[String]) -> Result { spokesperson_voice, spokesperson_rate, interruption_threshold, + listening_threshold, }) } @@ -6815,7 +6822,7 @@ fn create_stt_benchmark_report( &pack, config.runs, config.mode, - || create_input_runtime(&config.stt, None, &[]), + || create_input_runtime(&config.stt, None, &[], 0.5), ))) } @@ -7154,6 +7161,7 @@ fn create_input_runtime( config: &SttBackendConfig, endpoint_override: Option<&str>, _hints: &[String], + listening_threshold: f32, ) -> Result< ( VoiceInputRuntime, @@ -7219,7 +7227,7 @@ fn create_input_runtime( }; VoiceInputRuntime::start(VoiceInputConfig { engine, - speech_vad_threshold: 0.5, + speech_vad_threshold: listening_threshold, controls: VoiceInputControls::default(), }) .map_err(|error| match config { @@ -8507,6 +8515,30 @@ mod tests { use std::os::unix::net::UnixStream; use std::sync::Mutex; + #[test] + fn listening_threshold_is_independent_and_bounded() { + let base: Vec = ["berd-call", "session", "--tts-backend", "openai"] + .map(str::to_string) + .to_vec(); + assert_eq!(parse_args(&base).unwrap().listening_threshold, 0.5); + for value in ["0", "0.8", "1"] { + let mut args = base.clone(); + args.extend(["--listening-threshold".into(), value.into()]); + let config = parse_args(&args).unwrap(); + assert_eq!(config.interruption_threshold, 0.65); + assert_eq!(config.listening_threshold, value.parse::().unwrap()); + args.extend(["--interruption-threshold".into(), "0.7".into()]); + let config = parse_args(&args).unwrap(); + assert_eq!(config.interruption_threshold, 0.7); + assert_eq!(config.listening_threshold, value.parse::().unwrap()); + } + for value in ["-0.1", "1.1", "NaN", "inf", "quiet"] { + let mut args = base.clone(); + args.extend(["--listening-threshold".into(), value.into()]); + assert!(parse_args(&args).is_err(), "{value}"); + } + } + #[test] fn interruption_threshold_is_bounded_and_defaults_to_balanced() { let base = vec![ @@ -10816,6 +10848,7 @@ mod tests { spokesperson_voice: None, spokesperson_rate: None, interruption_threshold: 0.65, + listening_threshold: 0.5, } ); @@ -10831,6 +10864,7 @@ mod tests { spokesperson_voice: None, spokesperson_rate: None, interruption_threshold: 0.65, + listening_threshold: 0.5, } ); } @@ -10912,6 +10946,7 @@ mod tests { spokesperson_voice: None, spokesperson_rate: None, interruption_threshold: 0.65, + listening_threshold: 0.5, } ); assert!(parse_args(&args(&[ @@ -10972,6 +11007,7 @@ mod tests { spokesperson_voice: None, spokesperson_rate: None, interruption_threshold: 0.65, + listening_threshold: 0.5, }; let mut realtime = OpenAiSpokespersonConfig { endpoint: "ws://localhost".into(), @@ -11196,6 +11232,7 @@ mod tests { spokesperson_voice: None, spokesperson_rate: None, interruption_threshold: 0.65, + listening_threshold: 0.5, } ); assert!(parse_args(&args(&[ @@ -11262,6 +11299,7 @@ mod tests { spokesperson_voice: None, spokesperson_rate: None, interruption_threshold: 0.65, + listening_threshold: 0.5, } ); assert!(parse_args(&args(&[ @@ -11711,6 +11749,7 @@ mod tests { spokesperson_voice: None, spokesperson_rate: None, interruption_threshold: 0.65, + listening_threshold: 0.5, } ); } @@ -11942,6 +11981,60 @@ mod tests { ); } + #[test] + fn false_speech_hold_releases_only_after_speaking_and_recognition_settle() { + let mut core = SessionCore::default(); + let mut active = None; + let mut held = None; + let mut output = Vec::new(); + let mut next_token = 1; + for event in [ + VoiceInputEvent::RecognitionPendingChanged(true), + VoiceInputEvent::SpeakingChanged(true), + ] { + handle_voice_input_event(event, &mut core, &mut active, &mut next_token, &mut output) + .unwrap(); + } + process_prepare( + PrepareRequest { + id: 4, + acknowledgement: None, + text: "reply".into(), + }, + None, + &mut core, + &test_tts_slot(), + &test_input_policy_slot(), + &mut active, + &mut held, + &mut output, + ) + .unwrap(); + assert!(held.is_some()); + assert!(active.is_none()); + for pending in [true, false] { + let event = if pending { + VoiceInputEvent::SpeakingChanged(false) + } else { + VoiceInputEvent::RecognitionPendingChanged(false) + }; + handle_voice_input_event(event, &mut core, &mut active, &mut next_token, &mut output) + .unwrap(); + reevaluate_held( + &mut held, + &mut core, + Some(&test_tts_slot()), + Some(&test_input_policy_slot()), + &mut active, + &mut output, + ) + .unwrap(); + assert_eq!(held.is_some(), pending); + assert_eq!(active.is_some(), !pending); + } + assert_eq!(messages(&output).last().unwrap()["type"], "admitted"); + } + #[test] fn held_prepare_waits_for_pending_to_clear_without_a_timeout() { let mut core = SessionCore::default(); diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index 706878074..c69acbc4c 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -105,6 +105,7 @@ fn validate_arguments(arguments: &[String]) -> Result<(), String> { | "--spokesperson-voice" | "--spokesperson-rate" | "--interruption-threshold" + | "--listening-threshold" ) }) { @@ -232,6 +233,38 @@ pub(crate) fn with_spokesperson(arguments: &[String], on: bool, direct: bool) -> mod tests { use super::*; + #[test] + fn detection_thresholds_survive_reopen_and_override_independently() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let settings = SavedSettings { + arguments: [ + "--listening-threshold", + "0.8", + "--interruption-threshold", + "0.7", + ] + .map(str::to_string) + .to_vec(), + ..SavedSettings::default() + }; + save(&path, &settings).unwrap(); + let restored = load(&path).unwrap(); + assert_eq!(restored.arguments, settings.arguments); + assert_eq!( + merge_arguments( + &restored.arguments, + &["--interruption-threshold".into(), "0.9".into()] + ), + [ + "--listening-threshold", + "0.8", + "--interruption-threshold", + "0.9" + ] + ); + } + #[test] fn saved_settings_survive_reopen_and_explicit_backend_clears_dependent_flags() { let directory = tempfile::tempdir().unwrap(); From 3b385546513d2a4bacabff999b0e57d5660b8e69 Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Thu, 22 Oct 2026 07:15:50 -0400 Subject: [PATCH 04/10] berd-call: tune speech thresholds per audio setup Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/Cargo.lock | 1 + src-tauri/crates/berd-call/Cargo.toml | 1 + src-tauri/crates/berd-call/README.md | 10 + src-tauri/crates/berd-call/VAD_VISION.md | 11 + src-tauri/crates/berd-call/src/cli_help.rs | 4 + src-tauri/crates/berd-call/src/detection.rs | 294 ++++++++++++++++++ .../berd-call/src/detection_profiles.rs | 155 +++++++++ .../crates/berd-call/src/host_session.rs | 121 +++++++ src-tauri/crates/berd-call/src/input.rs | 159 +++++++++- src-tauri/crates/berd-call/src/lib.rs | 1 + .../crates/berd-call/src/macos_audio_route.rs | 50 ++- src-tauri/crates/berd-call/src/main.rs | 169 +++++++++- src-tauri/crates/berd-call/src/protocol.rs | 12 + .../crates/berd-call/src/saved_settings.rs | 2 + 14 files changed, 978 insertions(+), 12 deletions(-) create mode 100644 src-tauri/crates/berd-call/VAD_VISION.md create mode 100644 src-tauri/crates/berd-call/src/detection.rs create mode 100644 src-tauri/crates/berd-call/src/detection_profiles.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 2ba0c353a..a02b51ff7 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -601,6 +601,7 @@ dependencies = [ "objc2-app-kit", "objc2-avf-audio", "objc2-core-audio", + "objc2-core-foundation", "objc2-foundation", "ort", "ort-sys", diff --git a/src-tauri/crates/berd-call/Cargo.toml b/src-tauri/crates/berd-call/Cargo.toml index d02f9ca5f..dac0f6006 100644 --- a/src-tauri/crates/berd-call/Cargo.toml +++ b/src-tauri/crates/berd-call/Cargo.toml @@ -66,6 +66,7 @@ objc2 = "0.6" objc2-app-kit = "0.3.2" objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2"] } objc2-core-audio = "0.3.2" +objc2-core-foundation = "0.3.2" objc2-foundation = { version = "0.3.2", features = ["NSNotification", "NSOperation", "NSString", "block2"] } [target.'cfg(target_os = "macos")'.build-dependencies] diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 684090d9c..5b4ecaaaa 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -17,6 +17,16 @@ On macOS, a local speech detection also marks recognition pending. After local s Saved call arguments retain both values independently. To change a running call, supply its complete session configuration, for example `settings --restart --voice Aaron --language en-US --listening-threshold 0.8 --interruption-threshold 0.7`. Restart replaces the session options; include the existing backend, model, rate and other choices you want to retain. +### Experimental dynamic tuning and audio profiles + +With macOS speech recognition, `--dynamic-listening true` and `--dynamic-interruption true` independently opt into outcome-based tuning. Both default to false. Each enabled threshold moves one 0.01 tick after three unmatched observations, at most once every 30 seconds. Dynamic values start and remain within 0.10 to 0.95; manual thresholds still accept 0 to 1. + +A local detection that settles without a nonempty transcript suggests raising sensitivity's threshold. A nonempty transcript after at least five seconds without local detection suggests lowering it. Recognized speech clears false-trigger evidence. Split-result grace periods, mode changes, mute/reset epochs and device changes discard ambiguous evidence. These are experimental heuristics: recognition failure and playback leakage can still mislead them. They do not establish an optimal threshold or replace acoustic echo cancellation. + +The standalone macOS host saves applied thresholds and dynamic choices in `detection-profiles.json` beside its saved settings, keyed by the stable input/output device UID pair. It restores the pair on call startup and checks for device changes once per second. A new pair starts with defaults during a call. Profile files contain settings, not audio recordings or transcript text. An explicit profile edit takes precedence over a late automatic update. + +`berd-call detection-profile` reports the default audio setup and saved choices without starting capture. `berd-call detection-profile --settings '{"listening":{"value":0.70,"dynamic":true},"interruption":{"value":0.80,"dynamic":false}}'` saves choices for that pair. During a call, `berd-call status` exposes applied values and evidence counters in `session.detection`, plus `audioSetup`. Saving a profile alone affects its next restoration; the plugin restarts the call session when a user changes a control. Dynamic adjustments apply during the current session. + ## Command-line interface On macOS, install [Berd](https://github.com/block/berd/releases) first. Its diff --git a/src-tauri/crates/berd-call/VAD_VISION.md b/src-tauri/crates/berd-call/VAD_VISION.md new file mode 100644 index 000000000..d8e810f53 --- /dev/null +++ b/src-tauri/crates/berd-call/VAD_VISION.md @@ -0,0 +1,11 @@ +# Adaptive speech detection + +Proposed direction, October 22, 2026. Independent static thresholds are implemented by PR 421. Adaptive tuning and audio profiles below are proposed until their implementation is validated. + +A caller should be able to stop background noise holding replies without losing quiet speech. Listening and interruption each have a slider, visible current value, and an opt-in Dynamically adjust checkbox. Adjustments move one 0.01 tick at a time within 0.10 to 0.95. Manual values remain available across the full 0 to 1 range. + +For the initial experimental macOS recognizer, repeated local detections that settle without a nonempty transcript are evidence to increase the threshold. Nonempty transcripts without corresponding local detection provide evidence to decrease it. These are heuristics, not ground truth: recognizer failures, late or split results and playback leakage can mislead them. Ignore ambiguous observations across mute/reset, playback-mode and device changes; rate-limit changes and expose evidence counters with current values. Other recognition backends require their own reliable outcome signals before enabling adaptive tuning. + +Save independent values and dynamic choices by stable input/output device identity pair. AirPods input/output, another headset, and the built-in input/output pair retain separate profiles. Restoring a profile must discard old learning evidence so one setup cannot train another. No microphone recordings or transcript text are stored in these profiles. + +The host owns device identity and persistence. The shared input runtime owns thresholds, and the host-facing session protocol carries updates and reports applied values. Existing settings files remain readable by older binaries. Prefer a separate bounded, atomically replaced profile file to changing their strict schema. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 23e6d768e..6dba76e47 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -148,6 +148,7 @@ Usage: Each form also accepts: [--rate FLOAT] [--listening-threshold 0..1] [--interruption-threshold 0..1] + [--dynamic-listening true|false] [--dynamic-interruption true|false] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] [--stt-hint PHRASE]... [--mode conventional|expert-spokesperson] [--allow-direct-agent-speech] [--spokesperson-voice NAME] [--spokesperson-rate 0.25..1.5] @@ -156,6 +157,9 @@ Each form also accepts: --listening-threshold sets local speech detection before agent playback (default 0.5). --interruption-threshold sets local speech detection during agent playback (default 0.65). Higher values require stronger evidence of speech before pausing playback. +Dynamic adjustment is experimental, off by default, and requires macOS STT. +Each enabled threshold learns independently, moving by 0.01 within 0.10..0.95. +Use detection-profile [--settings JSON] to read or save the default audio setup profile. The host owns microphone capture, audio playback, transcript delivery, and agent integration. See PROTOCOL.md for the framed stdin, stdout, and PCM diff --git a/src-tauri/crates/berd-call/src/detection.rs b/src-tauri/crates/berd-call/src/detection.rs new file mode 100644 index 000000000..b533425eb --- /dev/null +++ b/src-tauri/crates/berd-call/src/detection.rs @@ -0,0 +1,294 @@ +//! Experimental outcome-based tuning of independent local speech thresholds. +use serde::{Deserialize, Serialize}; +use std::time::{Duration, Instant}; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct Threshold { + pub value: f32, + #[serde(default)] + pub dynamic: bool, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DetectionSettings { + pub listening: Threshold, + pub interruption: Threshold, +} + +impl Default for DetectionSettings { + fn default() -> Self { + Self { + listening: Threshold { + value: 0.5, + dynamic: false, + }, + interruption: Threshold { + value: 0.65, + dynamic: false, + }, + } + } +} + +impl DetectionSettings { + pub fn validate(self) -> Result { + for threshold in [self.listening, self.interruption] { + if !threshold.value.is_finite() || !(0.0..=1.0).contains(&threshold.value) { + return Err("speech thresholds must be finite and between 0 and 1".into()); + } + if threshold.dynamic && !(0.10..=0.95).contains(&threshold.value) { + return Err("dynamic speech thresholds must start between 0.10 and 0.95".into()); + } + } + Ok(self) + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +pub struct DetectionSnapshot { + pub settings: DetectionSettings, + pub adjustments: u64, + pub no_result: u64, + pub missed_detection: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DetectionMode { + Listening, + Interruption, +} + +const STEP: f32 = 0.01; +const COOLDOWN: Duration = Duration::from_secs(30); +const RESULT_GRACE: Duration = Duration::from_secs(5); + +pub struct DetectionTuner { + snapshot: DetectionSnapshot, + window: Option, + grace_until: Option, + evidence: [Evidence; 2], +} + +struct Observation { + mode: DetectionMode, + began: Instant, + detected: bool, + ambiguous: bool, +} + +#[derive(Default)] +struct Evidence { + no_result: u8, + missed: u8, + last_adjustment: Option, +} + +impl DetectionMode { + fn index(self) -> usize { + usize::from(self == Self::Interruption) + } +} + +impl DetectionTuner { + pub fn new(settings: DetectionSettings) -> Self { + Self { + snapshot: DetectionSnapshot { + settings, + adjustments: 0, + no_result: 0, + missed_detection: 0, + }, + window: None, + grace_until: None, + evidence: Default::default(), + } + } + pub fn snapshot(&self) -> DetectionSnapshot { + self.snapshot + } + pub fn observe(&mut self, mode: DetectionMode, detected: bool, now: Instant) { + let window = self.window.get_or_insert(Observation { + mode, + began: now, + detected: false, + ambiguous: false, + }); + window.ambiguous |= window.mode != mode; + window.detected |= detected; + } + pub fn final_transcript(&mut self, now: Instant) { + if let Some(window) = self.window.take().filter(|window| !window.ambiguous) { + let evidence = &mut self.evidence[window.mode.index()]; + evidence.no_result = 0; + if window.detected { + evidence.missed = 0; + } else if now.duration_since(window.began) >= RESULT_GRACE + && self.grace_until.is_none_or(|until| now >= until) + { + self.snapshot.missed_detection += 1; + evidence.missed = evidence.missed.saturating_add(1); + self.adjust(window.mode, false, now); + } + } + self.grace_until = Some(now + RESULT_GRACE); + } + pub fn no_result(&mut self, now: Instant) { + if let Some(window) = self + .window + .take() + .filter(|window| window.detected && !window.ambiguous) + { + self.snapshot.no_result += 1; + let evidence = &mut self.evidence[window.mode.index()]; + evidence.missed = 0; + evidence.no_result = evidence.no_result.saturating_add(1); + self.adjust(window.mode, true, now); + } + } + fn adjust(&mut self, mode: DetectionMode, increase: bool, now: Instant) { + let evidence = &mut self.evidence[mode.index()]; + let count = if increase { + evidence.no_result + } else { + evidence.missed + }; + if count < 3 + || evidence + .last_adjustment + .is_some_and(|last| now.duration_since(last) < COOLDOWN) + { + return; + } + let threshold = match mode { + DetectionMode::Listening => &mut self.snapshot.settings.listening, + DetectionMode::Interruption => &mut self.snapshot.settings.interruption, + }; + if !threshold.dynamic { + return; + } + // Round to slider ticks, and never jump a manual value toward a bound. + let next = + ((threshold.value + if increase { STEP } else { -STEP }) * 100.0).round() / 100.0; + if !(0.10..=0.95).contains(&next) { + return; + } + threshold.value = next; + self.snapshot.adjustments += 1; + evidence.last_adjustment = Some(now); + evidence.no_result = 0; + evidence.missed = 0; + } + pub fn reset_evidence(&mut self) { + self.window = None; + self.grace_until = None; + for evidence in &mut self.evidence { + evidence.no_result = 0; + evidence.missed = 0; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn tuner() -> DetectionTuner { + DetectionTuner::new(DetectionSettings { + listening: Threshold { + value: 0.5, + dynamic: true, + }, + interruption: Threshold { + value: 0.65, + dynamic: true, + }, + }) + } + #[test] + fn repeated_no_result_moves_only_the_detecting_mode_one_tick() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..3 { + let time = now + Duration::from_secs(n * 10); + tuner.observe(DetectionMode::Listening, true, time); + tuner.no_result(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings.listening.value, 0.51); + assert_eq!(tuner.snapshot().settings.interruption.value, 0.65); + assert_eq!(tuner.snapshot().adjustments, 1); + } + #[test] + fn manual_mode_never_learns() { + let settings = DetectionSettings::default(); + let mut tuner = DetectionTuner::new(settings); + let now = Instant::now(); + for n in 0..20 { + let time = now + Duration::from_secs(n * 40); + tuner.observe(DetectionMode::Interruption, true, time); + tuner.no_result(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings, settings); + } + #[test] + fn transcripts_without_detection_lower_only_the_observed_mode() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..3 { + let time = now + Duration::from_secs(n * 40); + tuner.observe(DetectionMode::Interruption, false, time); + tuner.final_transcript(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings.interruption.value, 0.64); + assert_eq!(tuner.snapshot().settings.listening.value, 0.5); + } + #[test] + fn adjustments_are_rate_limited_and_stay_inside_dynamic_bounds() { + let mut tuner = tuner(); + tuner.snapshot.settings.listening.value = 0.94; + let now = Instant::now(); + for n in 0..100 { + let time = now + Duration::from_secs(n * 7); + tuner.observe(DetectionMode::Listening, true, time); + tuner.no_result(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings.listening.value, 0.95); + assert_eq!(tuner.snapshot().adjustments, 1); + tuner.snapshot.settings.listening.value = 0.5; + tuner.reset_evidence(); + for n in 0..6 { + let time = now + Duration::from_secs(n); + tuner.observe(DetectionMode::Listening, true, time); + tuner.no_result(time); + } + assert_eq!(tuner.snapshot().adjustments, 2); + } + + #[test] + fn recent_split_finals_and_detected_speech_do_not_lower_thresholds() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..20 { + let time = now + Duration::from_secs(n); + tuner.observe(DetectionMode::Listening, n == 0, time); + tuner.final_transcript(time); + } + assert_eq!(tuner.snapshot().adjustments, 0); + } + + #[test] + fn mode_switch_and_reset_discard_ambiguous_evidence() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..10 { + let time = now + Duration::from_secs(n * 40); + tuner.observe(DetectionMode::Listening, true, time); + tuner.observe(DetectionMode::Interruption, false, time); + tuner.no_result(time + Duration::from_secs(6)); + tuner.observe(DetectionMode::Listening, true, time); + tuner.reset_evidence(); + tuner.no_result(time + Duration::from_secs(6)); + } + assert_eq!(tuner.snapshot().settings, self::tuner().snapshot().settings); + } +} diff --git a/src-tauri/crates/berd-call/src/detection_profiles.rs b/src-tauri/crates/berd-call/src/detection_profiles.rs new file mode 100644 index 000000000..b48075143 --- /dev/null +++ b/src-tauri/crates/berd-call/src/detection_profiles.rs @@ -0,0 +1,155 @@ +use berd_call::{detection::DetectionSettings, macos_audio_route::AudioSetup}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +#[derive(Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Profiles { + profiles: BTreeMap, +} + +pub fn path() -> Result { + Ok(crate::saved_settings::path()?.with_file_name("detection-profiles.json")) +} + +fn key(setup: &AudioSetup) -> String { + serde_json::to_string(&[&setup.input_uid, &setup.output_uid]).expect("device strings serialize") +} + +fn read(path: &Path) -> Result { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Profiles::default()) + } + Err(error) => return Err(error.to_string()), + }; + let mut bytes = Vec::new(); + file.take(65537) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + if bytes.len() > 65536 { + return Err("speech profiles exceed 64 KiB".into()); + } + let profiles: Profiles = serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + if profiles.profiles.len() > 64 { + return Err("too many speech profiles".into()); + } + for settings in profiles.profiles.values() { + settings.validate()?; + } + Ok(profiles) +} + +pub fn load(path: &Path, setup: &AudioSetup) -> Result, String> { + Ok(read(path)?.profiles.get(&key(setup)).copied()) +} + +pub fn save(path: &Path, setup: &AudioSetup, settings: DetectionSettings) -> Result<(), String> { + write(path, setup, settings, None).map(|_| ()) +} + +pub fn save_if_unchanged( + path: &Path, + setup: &AudioSetup, + expected: Option, + settings: DetectionSettings, +) -> Result { + write(path, setup, settings, Some(expected)) +} + +fn write( + path: &Path, + setup: &AudioSetup, + settings: DetectionSettings, + expected: Option>, +) -> Result { + settings.validate()?; + let parent = path + .parent() + .ok_or("speech profiles need a parent directory")?; + std::fs::create_dir_all(parent).map_err(|error| error.to_string())?; + let lock = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path.with_extension("json.lock")) + .map_err(|error| error.to_string())?; + fs2::FileExt::lock_exclusive(&lock).map_err(|error| error.to_string())?; + let mut profiles = read(path)?; + if expected.is_some_and(|expected| profiles.profiles.get(&key(setup)).copied() != expected) { + return Ok(false); + } + profiles.profiles.insert(key(setup), settings); + if profiles.profiles.len() > 64 { + return Err("too many speech profiles".into()); + } + let bytes = serde_json::to_vec_pretty(&profiles).map_err(|error| error.to_string())?; + if bytes.len() > 65536 { + return Err("speech profiles exceed 64 KiB".into()); + } + let mut file = tempfile::NamedTempFile::new_in(parent).map_err(|error| error.to_string())?; + file.write_all(&bytes) + .and_then(|()| file.as_file().sync_all()) + .map_err(|error| error.to_string())?; + file.persist(path).map_err(|error| error.to_string())?; + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + fn setup(input: &str, output: &str) -> AudioSetup { + AudioSetup { + input_uid: input.into(), + output_uid: output.into(), + input_name: "same display name".into(), + output_name: "same display name".into(), + } + } + #[test] + fn late_learning_cannot_overwrite_an_explicit_profile_edit() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("profiles.json"); + let setup = setup("mic", "output"); + let original = DetectionSettings::default(); + save(&path, &setup, original).unwrap(); + let mut edited = original; + edited.listening.value = 0.77; + save(&path, &setup, edited).unwrap(); + let mut learned = original; + learned.listening.value = 0.51; + assert!(!save_if_unchanged(&path, &setup, Some(original), learned).unwrap()); + assert_eq!(load(&path, &setup).unwrap(), Some(edited)); + assert!(save_if_unchanged(&path, &setup, Some(edited), learned).unwrap()); + } + + #[test] + fn profiles_are_isolated_by_both_stable_device_ids_and_survive_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("profiles.json"); + let airpods = setup("airpods-mic", "airpods-output"); + let mixed = setup("mac-mic", "airpods-output"); + let mut first = DetectionSettings::default(); + first.listening.value = 0.71; + first.listening.dynamic = true; + let mut second = DetectionSettings::default(); + second.interruption.value = 0.83; + save(&path, &airpods, first).unwrap(); + save(&path, &mixed, second).unwrap(); + assert_eq!(load(&path, &airpods).unwrap(), Some(first)); + assert_eq!(load(&path, &mixed).unwrap(), Some(second)); + assert_eq!( + load(&path, &setup("headset-mic", "headset-output")).unwrap(), + None + ); + assert!(!std::fs::read_to_string(path) + .unwrap() + .contains("same display name")); + } +} diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 9f866f71b..210d8191f 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -435,6 +435,7 @@ fn start_session( .store(muted.load(Ordering::SeqCst), Ordering::SeqCst); actor.capture_muted = muted; actor.capture_mute_epoch = mute_epoch; + actor.restore_detection_profile()?; actor.system_input_mute = system_input_mute; if actor.system_input_mute.is_none() { match crate::system_input_mute::SystemInputMute::install( @@ -912,6 +913,7 @@ impl HostControl for SessionControl { "statusSounds": sounds.mode, "conversationStatus": sounds.status, "sessionArguments": &self.session_arguments[1..], + "audioSetup": berd_call::macos_audio_route::default_audio_setup(), })) } @@ -987,7 +989,16 @@ enum SavedPreference { StatusSounds(berd_call::StatusSoundMode), } +struct ActiveDetectionProfile { + persisted: Option, + setup: berd_call::macos_audio_route::AudioSetup, + path: std::path::PathBuf, + next_check: Instant, + pending: Option, +} + struct SessionActor { + detection_profile: Option, saved: Option<(std::path::PathBuf, crate::saved_settings::SavedSettings)>, input_speaking: bool, recognition_pending: bool, @@ -1098,6 +1109,7 @@ impl SessionActor { audio_commands, next_id: 2, pending_speak: None, + detection_profile: None, pending_settings: std::collections::HashMap::new(), spokesperson_settings: HashSet::new(), waiting_speaks: VecDeque::new(), @@ -1178,8 +1190,77 @@ impl SessionActor { } } + fn restore_detection_profile(&mut self) -> Result<(), String> { + let detection = self + .session + .lock() + .map_err(|_| "session settings lock failed")? + .detection; + let Some(snapshot) = detection else { + return Ok(()); + }; + let Some(setup) = berd_call::macos_audio_route::default_audio_setup() else { + return Ok(()); + }; + let path = crate::detection_profiles::path()?; + let persisted = crate::detection_profiles::load(&path, &setup)?; + let settings = persisted.unwrap_or(snapshot.settings); + let id = self.next_id(); + send_request( + &self.writer, + &SessionRequest::SetDetectionSettings { id, settings }, + )?; + self.detection_profile = Some(ActiveDetectionProfile { + persisted, + setup, + path, + next_check: Instant::now() + Duration::from_secs(1), + pending: Some(id), + }); + Ok(()) + } + + fn poll_detection_profile(&mut self) -> Result<(), String> { + let Some(profile) = self.detection_profile.as_mut() else { + return Ok(()); + }; + if profile.pending.is_some() || Instant::now() < profile.next_check { + return Ok(()); + } + profile.next_check = Instant::now() + Duration::from_secs(1); + let Some(setup) = berd_call::macos_audio_route::default_audio_setup() else { + return Ok(()); + }; + if setup.input_uid == profile.setup.input_uid + && setup.output_uid == profile.setup.output_uid + { + return Ok(()); + } + let settings = match crate::detection_profiles::load(&profile.path, &setup) { + Ok(settings) => settings, + Err(error) => { + eprintln!("berd-call: could not restore speech profile: {error}"); + return Ok(()); + } + }; + profile.persisted = settings; + let settings = settings.unwrap_or_default(); + profile.setup = setup; + let id = self.next_id(); + send_request( + &self.writer, + &SessionRequest::SetDetectionSettings { id, settings }, + )?; + self.detection_profile + .as_mut() + .expect("profile exists") + .pending = Some(id); + Ok(()) + } + fn poll(&mut self) -> Result<(), String> { if !self.stopping { + self.poll_detection_profile()?; if let Some(muted) = self .system_input_mute .as_mut() @@ -1440,6 +1521,45 @@ impl SessionActor { fn handle_event(&mut self, event: SessionMessage) -> Result<(), String> { match event { + SessionMessage::DetectionSettingsResult { + id, + snapshot, + message, + } => { + if let Some(profile) = self.detection_profile.as_mut() { + if profile.pending.is_some() && profile.pending != id { + return Ok(()); + } + if profile.pending == id { + profile.pending = None; + } + if let Some(snapshot) = snapshot { + // A device switch observed before its ordered acknowledgement + // cannot save the old setup's learning under the new pair. + if berd_call::macos_audio_route::default_audio_setup().is_some_and( + |setup| { + setup.input_uid == profile.setup.input_uid + && setup.output_uid == profile.setup.output_uid + }, + ) { + match crate::detection_profiles::save_if_unchanged(&profile.path, &profile.setup, profile.persisted, snapshot.settings) { + Ok(true) => profile.persisted = Some(snapshot.settings), + Ok(false) => {}, + Err(error) => eprintln!("berd-call: speech detection applied but could not be saved: {error}"), + } + } + } + } + if let Some(snapshot) = snapshot { + self.session + .lock() + .map_err(|_| "session settings lock failed")? + .detection = Some(snapshot); + } + if let Some(message) = message { + eprintln!("berd-call: speech detection update rejected: {message}"); + } + } SessionMessage::InputSpeaking { active } => self.input_speaking = active, SessionMessage::RecognitionPending { active } => self.recognition_pending = active, SessionMessage::State { @@ -3761,6 +3881,7 @@ mod tests { fn test_snapshot() -> VoiceSessionSnapshot { VoiceSessionSnapshot { + detection: None, spokesperson: None, tts: berd_call::TtsConfigurationSnapshot { revision: 1, diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index d1e48dadd..0d6e51410 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -162,6 +162,7 @@ pub struct VoiceInputControls { } struct ControlState { + detection: Mutex>, transition: Mutex, muted: AtomicBool, mute_epoch: AtomicU64, @@ -244,6 +245,7 @@ impl Default for VoiceInputControls { fn default() -> Self { Self { shared: Arc::new(ControlState { + detection: Mutex::new(None), transition: Mutex::new(ControlTransitionState::default()), muted: AtomicBool::new(false), mute_epoch: AtomicU64::new(0), @@ -255,6 +257,63 @@ impl Default for VoiceInputControls { } impl VoiceInputControls { + pub fn set_detection( + &self, + settings: crate::detection::DetectionSettings, + ) -> Result<(), String> { + let settings = settings.validate()?; + let mut detection = self + .shared + .detection + .lock() + .map_err(|_| "speech detection lock failed")?; + self.reset(); + *detection = Some(crate::detection::DetectionTuner::new(settings)); + Ok(()) + } + + pub fn detection(&self) -> Option { + self.shared + .detection + .lock() + .ok()? + .as_ref() + .map(|tuner| tuner.snapshot()) + } + + #[cfg(target_os = "macos")] + fn observe_detection(&self, detected: bool, epoch: u64) { + let mode = if self.shared.assistant_speaking.load(Ordering::Acquire) { + crate::detection::DetectionMode::Interruption + } else { + crate::detection::DetectionMode::Listening + }; + if let Ok(mut detection) = self.shared.detection.lock() { + if self.mute_epoch() != epoch { + return; + } + if let Some(tuner) = detection.as_mut() { + tuner.observe(mode, detected, Instant::now()); + } + } + } + + #[cfg(target_os = "macos")] + fn detection_result(&self, recognized: bool, epoch: u64) { + if let Ok(mut detection) = self.shared.detection.lock() { + if self.mute_epoch() != epoch { + return; + } + if let Some(tuner) = detection.as_mut() { + if recognized { + tuner.final_transcript(Instant::now()); + } else { + tuner.no_result(Instant::now()); + } + } + } + } + pub fn is_muted(&self) -> bool { self.shared.muted.load(Ordering::Acquire) } @@ -329,6 +388,13 @@ impl VoiceInputControls { } fn vad_threshold(&self, speech_threshold: f32) -> f32 { + if let Some(snapshot) = self.detection() { + return if self.shared.assistant_speaking.load(Ordering::Acquire) { + snapshot.settings.interruption.value + } else { + snapshot.settings.listening.value + }; + } if self.shared.assistant_speaking.load(Ordering::Acquire) { f32::from_bits(self.shared.assistant_vad_threshold.load(Ordering::Acquire)) } else { @@ -1300,7 +1366,7 @@ fn new_mac_recognizer( } #[cfg(target_os = "macos")] -fn forward_mac_events( +fn forward_mac_events_with_detection( recognition_events: &mut tokio_mpsc::UnboundedReceiver< crate::mac_speech::MacSpeechRecognitionEvent, >, @@ -1308,13 +1374,19 @@ fn forward_mac_events( pending: &mut PendingRecognitions, settle_deadline: &mut Option, storage_deadline: Option, + controls: Option<(&VoiceInputControls, u64)>, ) -> Result<(), ()> { while let Ok(event) = recognition_events.try_recv() { match event { crate::mac_speech::MacSpeechRecognitionEvent::Final(text) => { let text = text.trim().to_string(); if !text.is_empty() { + if let Some((controls, epoch)) = controls { + controls.detection_result(true, epoch); + } send_final(events, text, storage_deadline)?; + } else if let Some((controls, epoch)) = controls { + controls.detection_result(false, epoch); } pending.resolve(events)?; if pending.count == 0 { @@ -1347,6 +1419,26 @@ fn forward_mac_events( Ok(()) } +#[cfg(all(test, target_os = "macos"))] +fn forward_mac_events( + recognition_events: &mut tokio_mpsc::UnboundedReceiver< + crate::mac_speech::MacSpeechRecognitionEvent, + >, + events: &tokio_mpsc::Sender, + pending: &mut PendingRecognitions, + settle_deadline: &mut Option, + storage_deadline: Option, +) -> Result<(), ()> { + forward_mac_events_with_detection( + recognition_events, + events, + pending, + settle_deadline, + storage_deadline, + None, + ) +} + #[cfg(target_os = "macos")] fn begin_mac_turn( pending: &mut PendingRecognitions, @@ -1424,18 +1516,20 @@ fn mac_speech_worker( } loop { - if forward_mac_events( + if forward_mac_events_with_detection( &mut recognition_events, &events, &mut pending, &mut settle_deadline, None, + Some((&controls, observed_epoch)), ) .is_err() { return; } if mac_settle_expired(&pending, settle_deadline, Instant::now()) { + controls.detection_result(false, observed_epoch); // SpeechTranscriber has no per-utterance no-result callback. End // this recognizer generation at the bounded idle deadline so its // suppressed late callbacks cannot resolve a later generation. @@ -1467,6 +1561,11 @@ fn mac_speech_worker( effective_mute_epoch(&controls, &shutdown, &shutdown_mute_epoch); if current_epoch != observed_epoch { observed_epoch = current_epoch; + if let Ok(mut detection) = controls.shared.detection.lock() { + if let Some(tuner) = detection.as_mut() { + tuner.reset_evidence(); + } + } input_48k.clear(); leftover_16k.clear(); silence_frames = 0; @@ -1515,7 +1614,10 @@ fn mac_speech_worker( while leftover_16k.len() >= VAD_FRAME_SAMPLES { let frame: Vec = leftover_16k.drain(..VAD_FRAME_SAMPLES).collect(); let clamped = clamp_vad_frame(&frame); - if vad.predict_f32(&clamped) > controls.vad_threshold(speech_vad_threshold) { + let detected = + vad.predict_f32(&clamped) > controls.vad_threshold(speech_vad_threshold); + controls.observe_detection(detected, observed_epoch); + if detected { silence_frames = 0; if !in_speech { in_speech = true; @@ -1535,12 +1637,13 @@ fn mac_speech_worker( } } } - if forward_mac_events( + if forward_mac_events_with_detection( &mut recognition_events, &events, &mut pending, &mut settle_deadline, None, + Some((&controls, observed_epoch)), ) .is_err() { @@ -1562,12 +1665,13 @@ fn mac_speech_worker( return; } let deadline = Instant::now() + FINAL_STORAGE_TIMEOUT; - let _ = forward_mac_events( + let _ = forward_mac_events_with_detection( &mut recognition_events, &events, &mut pending, &mut settle_deadline, Some(deadline), + Some((&controls, observed_epoch)), ); let _ = pending.reset(&events); } @@ -3051,6 +3155,51 @@ mod tests { assert!(!finished.discard_on_shutdown.load(Ordering::Acquire)); } + #[cfg(target_os = "macos")] + #[test] + fn stale_input_generation_cannot_train_a_new_profile() { + let controls = VoiceInputControls::default(); + let old_epoch = controls.mute_epoch(); + let mut settings = crate::detection::DetectionSettings::default(); + settings.listening.dynamic = true; + controls.set_detection(settings).unwrap(); + for _ in 0..10 { + controls.observe_detection(true, old_epoch); + controls.detection_result(false, old_epoch); + } + let snapshot = controls.detection().unwrap(); + assert_eq!(snapshot.settings, settings); + assert_eq!(snapshot.no_result, 0); + assert_eq!(snapshot.adjustments, 0); + } + + #[test] + fn applied_detection_settings_survive_playback_guards_and_reject_invalid_updates() { + let controls = VoiceInputControls::default(); + let settings = crate::detection::DetectionSettings { + listening: crate::detection::Threshold { + value: 0.81, + dynamic: true, + }, + interruption: crate::detection::Threshold { + value: 0.72, + dynamic: false, + }, + }; + controls.set_detection(settings).unwrap(); + assert_eq!(controls.vad_threshold(0.5), 0.81); + let playback = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + assert_eq!(controls.vad_threshold(0.5), 0.72); + drop(playback); + assert_eq!(controls.vad_threshold(0.5), 0.81); + let mut invalid = settings; + invalid.listening.value = f32::NAN; + assert!(controls.set_detection(invalid).is_err()); + assert_eq!(controls.detection().unwrap().settings, settings); + } + #[test] fn listening_threshold_applies_before_and_after_independent_playback_threshold() { let controls = VoiceInputControls::default(); diff --git a/src-tauri/crates/berd-call/src/lib.rs b/src-tauri/crates/berd-call/src/lib.rs index f19eff088..76c3d2e30 100644 --- a/src-tauri/crates/berd-call/src/lib.rs +++ b/src-tauri/crates/berd-call/src/lib.rs @@ -8,6 +8,7 @@ mod audio_output; pub mod benchmark; pub mod causal_inbox; mod configured_tts; +pub mod detection; pub mod endpoint_url; pub mod expert_spokesperson; pub mod input; diff --git a/src-tauri/crates/berd-call/src/macos_audio_route.rs b/src-tauri/crates/berd-call/src/macos_audio_route.rs index 1514a0d09..bf3501bdc 100644 --- a/src-tauri/crates/berd-call/src/macos_audio_route.rs +++ b/src-tauri/crates/berd-call/src/macos_audio_route.rs @@ -3,7 +3,7 @@ use std::ptr::{null, NonNull}; use std::time::Duration; use coreaudio::audio_unit::macos_helpers::{ - get_default_device_id, get_device_id_from_name, get_device_transport_type, + get_default_device_id, get_device_id_from_name, get_device_name, get_device_transport_type, }; use objc2_core_audio::{ kAudioDevicePropertyScopeOutput, kAudioDevicePropertyStreams, kAudioDeviceTransportTypeBuiltIn, @@ -161,3 +161,51 @@ mod tests { )); } } + +/// Stable default-device identities used to isolate local speech calibration. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +pub struct AudioSetup { + pub input_uid: String, + pub output_uid: String, + pub input_name: String, + pub output_name: String, +} + +pub fn default_audio_setup() -> Option { + fn uid(device: AudioObjectID) -> Option { + use objc2_core_audio::kAudioDevicePropertyDeviceUID; + use objc2_core_foundation::{CFRetained, CFString}; + let address = AudioObjectPropertyAddress { + mSelector: kAudioDevicePropertyDeviceUID, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, + }; + let mut value: *const CFString = null(); + let mut size = mem::size_of_val(&value) as u32; + // SAFETY: CoreAudio writes one retained CFString pointer into a correctly + // sized live buffer. CFRetained owns and releases the returned reference. + let status = unsafe { + AudioObjectGetPropertyData( + device, + NonNull::from(&address), + 0, + null(), + NonNull::from(&mut size), + NonNull::from(&mut value).cast(), + ) + }; + if status != kAudioHardwareNoError { + return None; + } + let pointer = NonNull::new(value.cast_mut())?; + Some(unsafe { CFRetained::from_raw(pointer) }.to_string()) + } + let input = get_default_device_id(true)?; + let output = get_default_device_id(false)?; + Some(AudioSetup { + input_uid: uid(input)?, + output_uid: uid(output)?, + input_name: get_device_name(input).ok()?, + output_name: get_device_name(output).ok()?, + }) +} diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 0c64188c4..634734683 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -66,6 +66,8 @@ use serde::Serialize; mod cli_help; #[cfg(target_os = "macos")] mod codex; +#[cfg(target_os = "macos")] +mod detection_profiles; mod host_control; #[cfg(target_os = "macos")] mod host_session; @@ -208,6 +210,8 @@ struct SessionConfig { spokesperson_rate: Option, interruption_threshold: f32, listening_threshold: f32, + dynamic_listening: bool, + dynamic_interruption: bool, } #[derive(Clone, Debug, Default, PartialEq)] @@ -526,6 +530,32 @@ fn main() { None => {} } match args.get(1).map(String::as_str) { + #[cfg(target_os = "macos")] + Some("detection-profile") => { + let result = (|| -> Result { + let setup = berd_call::macos_audio_route::default_audio_setup() + .ok_or("audio setup is unavailable")?; + let path = detection_profiles::path()?; + if args.len() > 2 { + if args.len() != 4 || args[2] != "--settings" { + return Err("usage: berd-call detection-profile [--settings JSON]".into()); + } + let settings: berd_call::detection::DetectionSettings = + serde_json::from_str(&args[3]).map_err(|error| error.to_string())?; + detection_profiles::save(&path, &setup, settings)?; + } + Ok( + serde_json::json!({"audioSetup": setup, "settings": detection_profiles::load(&path, &setup)?.unwrap_or_default()}), + ) + })(); + match result { + Ok(value) => println!("{value}"), + Err(error) => { + eprintln!("berd-call detection-profile failed: {error}"); + std::process::exit(1); + } + } + } Some("start") => { let options = parse_or_exit(parse_saved_start_args(&args), &args); #[cfg(target_os = "macos")] @@ -2090,6 +2120,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let mut next_tts_update_attempt = 1_u64; let mut input_runtime: Option = None; let mut input_events: Option> = None; + let mut last_detection = None; let mut input_controls: Option = None; let mut next_input_token = 1_u64; let mut pending_control = None; @@ -2144,6 +2175,22 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String &mut tts_update, &mut writer, )?; + if let Some(snapshot) = input_controls + .as_ref() + .and_then(|controls| controls.detection()) + { + if last_detection != Some(snapshot) { + write_message( + &mut writer, + &SessionMessage::DetectionSettingsResult { + id: None, + snapshot: Some(snapshot), + message: None, + }, + )?; + last_detection = Some(snapshot); + } + } reevaluate_held( &mut held, &mut core, @@ -2300,7 +2347,16 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String &config.stt, config.endpoints.stt.as_deref(), &config.stt_hints, - config.listening_threshold, + berd_call::detection::DetectionSettings { + listening: berd_call::detection::Threshold { + value: config.listening_threshold, + dynamic: config.dynamic_listening, + }, + interruption: berd_call::detection::Threshold { + value: config.interruption_threshold, + dynamic: config.dynamic_interruption, + }, + }, ) { Ok(runtime) => runtime, Err(message) => { @@ -2332,10 +2388,14 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String status_sound_runtime.set_output_device(status_sound_output_device); let input_policy = InputDuringTtsSlot::new(input_during_tts); let session = VoiceSessionSnapshot { + detection: input_controls + .as_ref() + .and_then(|controls| controls.detection()), tts: slot.snapshot()?, input_during_tts: input_policy.snapshot()?, spokesperson: None, }; + last_detection = session.detection; tts_slot = Some(slot); input_during_tts_slot = Some(input_policy); write_message( @@ -2436,6 +2496,36 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String }); } } + Input::Request(SessionRequest::SetDetectionSettings { id, settings }) => { + let result = if (settings.listening.dynamic || settings.interruption.dynamic) + && !matches!(config.stt, SttBackendConfig::Macos) + { + Err( + "dynamic speech detection currently requires macOS speech recognition" + .to_string(), + ) + } else { + input_controls + .as_ref() + .ok_or_else(|| "speech input is not ready".to_string()) + .and_then(|controls| controls.set_detection(settings)) + }; + let snapshot = result + .as_ref() + .ok() + .and_then(|_| input_controls.as_ref()?.detection()); + if snapshot.is_some() { + last_detection = snapshot; + } + write_message( + &mut writer, + &SessionMessage::DetectionSettingsResult { + id: Some(id), + snapshot, + message: result.err(), + }, + )?; + } Input::Request(SessionRequest::SetInputDuringTts { id, expected_revision, @@ -4578,6 +4668,7 @@ fn run_expert_spokesperson_session( }; let input_policy = InputDuringTtsSlot::new(input_during_tts); let snapshot = VoiceSessionSnapshot { + detection: None, tts: match &reply_tts { Some(slot) => slot.snapshot()?, None => tts.clone(), @@ -5080,6 +5171,18 @@ fn run_expert_spokesperson_session( &mut writer, )?; } + Input::Request(SessionRequest::SetDetectionSettings { id, .. }) => { + write_message( + &mut writer, + &SessionMessage::DetectionSettingsResult { + id: Some(id), + snapshot: None, + message: Some( + "local speech detection controls require a conventional call".into(), + ), + }, + )?; + } Input::Request(SessionRequest::SetInputDuringTts { id, expected_revision, @@ -6081,6 +6184,8 @@ fn parse_args(args: &[String]) -> Result { let mut spokesperson_rate = None; let mut interruption_threshold = 0.65; let mut listening_threshold = 0.5; + let mut dynamic_listening = false; + let mut dynamic_interruption = false; let mut index = 2; while index < args.len() { let flag = args[index].as_str(); @@ -6109,6 +6214,16 @@ fn parse_args(args: &[String]) -> Result { interruption_threshold = threshold; } } + "--dynamic-listening" | "--dynamic-interruption" => { + let enabled = value + .parse::() + .map_err(|_| format!("{flag} must be true or false"))?; + if flag == "--dynamic-listening" { + dynamic_listening = enabled; + } else { + dynamic_interruption = enabled; + } + } "--tts-backend" => backend = value, "--voice" => voice = Some(value.clone()), "--language" => language = Some(value.clone()), @@ -6203,6 +6318,8 @@ fn parse_args(args: &[String]) -> Result { spokesperson_rate, interruption_threshold, listening_threshold, + dynamic_listening, + dynamic_interruption, }) } @@ -6822,7 +6939,7 @@ fn create_stt_benchmark_report( &pack, config.runs, config.mode, - || create_input_runtime(&config.stt, None, &[], 0.5), + || create_input_runtime(&config.stt, None, &[], Default::default()), ))) } @@ -7161,7 +7278,7 @@ fn create_input_runtime( config: &SttBackendConfig, endpoint_override: Option<&str>, _hints: &[String], - listening_threshold: f32, + detection: berd_call::detection::DetectionSettings, ) -> Result< ( VoiceInputRuntime, @@ -7225,10 +7342,17 @@ fn create_input_runtime( } } }; + if (detection.listening.dynamic || detection.interruption.dynamic) + && !matches!(config, SttBackendConfig::Macos) + { + return Err("dynamic speech detection currently requires macOS speech recognition".into()); + } + let controls = VoiceInputControls::default(); + controls.set_detection(detection)?; VoiceInputRuntime::start(VoiceInputConfig { engine, - speech_vad_threshold: listening_threshold, - controls: VoiceInputControls::default(), + speech_vad_threshold: detection.listening.value, + controls, }) .map_err(|error| match config { SttBackendConfig::Macos => format!( @@ -8229,7 +8353,8 @@ fn decode_framed_input(reader: &mut impl Read, header: [u8; FRAME_HEADER_BYTES]) fn validate_request(request: SessionRequest) -> Result { let id = match &request { - SessionRequest::Hello { id, .. } + SessionRequest::SetDetectionSettings { id, .. } + | SessionRequest::Hello { id, .. } | SessionRequest::SetInputMuted { id, .. } | SessionRequest::SetConversationStatus { id, .. } | SessionRequest::SetTtsSettings { id, .. } @@ -8515,6 +8640,23 @@ mod tests { use std::os::unix::net::UnixStream; use std::sync::Mutex; + #[test] + fn dynamic_detection_switches_are_explicit_and_independent() { + let base: Vec = ["berd-call", "session", "--tts-backend", "openai"] + .map(str::to_string) + .to_vec(); + let config = parse_args(&base).unwrap(); + assert!(!config.dynamic_listening && !config.dynamic_interruption); + let mut args = base.clone(); + args.extend(["--dynamic-listening".into(), "true".into()]); + let config = parse_args(&args).unwrap(); + assert!(config.dynamic_listening && !config.dynamic_interruption); + args.extend(["--dynamic-interruption".into(), "true".into()]); + assert!(parse_args(&args).unwrap().dynamic_interruption); + args.extend(["--dynamic-listening".into(), "maybe".into()]); + assert!(parse_args(&args).is_err()); + } + #[test] fn listening_threshold_is_independent_and_bounded() { let base: Vec = ["berd-call", "session", "--tts-backend", "openai"] @@ -10252,6 +10394,7 @@ mod tests { id: 1, protocol: SESSION_PROTOCOL_VERSION, session: VoiceSessionSnapshot { + detection: None, tts: snapshot.clone(), input_during_tts: test_input_policy(), spokesperson: None, @@ -10849,6 +10992,8 @@ mod tests { spokesperson_rate: None, interruption_threshold: 0.65, listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, } ); @@ -10865,6 +11010,8 @@ mod tests { spokesperson_rate: None, interruption_threshold: 0.65, listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, } ); } @@ -10947,6 +11094,8 @@ mod tests { spokesperson_rate: None, interruption_threshold: 0.65, listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, } ); assert!(parse_args(&args(&[ @@ -11008,6 +11157,8 @@ mod tests { spokesperson_rate: None, interruption_threshold: 0.65, listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, }; let mut realtime = OpenAiSpokespersonConfig { endpoint: "ws://localhost".into(), @@ -11233,6 +11384,8 @@ mod tests { spokesperson_rate: None, interruption_threshold: 0.65, listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, } ); assert!(parse_args(&args(&[ @@ -11300,6 +11453,8 @@ mod tests { spokesperson_rate: None, interruption_threshold: 0.65, listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, } ); assert!(parse_args(&args(&[ @@ -11750,6 +11905,8 @@ mod tests { spokesperson_rate: None, interruption_threshold: 0.65, listening_threshold: 0.5, + dynamic_listening: false, + dynamic_interruption: false, } ); } diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index cf6d578d5..d489afc78 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -9,6 +9,10 @@ use crate::{ #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] pub enum SessionRequest { + SetDetectionSettings { + id: u64, + settings: crate::detection::DetectionSettings, + }, Hello { id: u64, input_during_tts: InputDuringTtsPolicy, @@ -173,6 +177,8 @@ pub enum OutputReadyOutcome { pub struct VoiceSessionSnapshot { /// The voice that speaks replies: the conventional backend, or the /// Spokesperson's own voice when it reads them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detection: Option, pub tts: TtsConfigurationSnapshot, pub input_during_tts: InputDuringTtsSnapshot, /// The Spokesperson's voice, when replies have their own. @@ -213,6 +219,11 @@ pub enum ExpertTurnOutcome { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum SessionMessage { + DetectionSettingsResult { + id: Option, + snapshot: Option, + message: Option, + }, Ready { id: u64, protocol: u32, @@ -412,6 +423,7 @@ mod tests { id: 4, protocol: 5, session: VoiceSessionSnapshot { + detection: None, tts: TtsConfigurationSnapshot { revision: 1, settings: TtsSettings::OpenAi { diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index c69acbc4c..9df0dfdfc 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -106,6 +106,8 @@ fn validate_arguments(arguments: &[String]) -> Result<(), String> { | "--spokesperson-rate" | "--interruption-threshold" | "--listening-threshold" + | "--dynamic-listening" + | "--dynamic-interruption" ) }) { From ca1db13323eec589752981ec20106d673034f728 Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Thu, 22 Oct 2026 07:18:40 -0400 Subject: [PATCH 05/10] berd-call: retain adaptive learning across idle playback changes Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/src/detection.rs | 25 +++++++++++++++++++ .../berd-call/src/detection_profiles.rs | 3 +++ 2 files changed, 28 insertions(+) diff --git a/src-tauri/crates/berd-call/src/detection.rs b/src-tauri/crates/berd-call/src/detection.rs index b533425eb..46effb229 100644 --- a/src-tauri/crates/berd-call/src/detection.rs +++ b/src-tauri/crates/berd-call/src/detection.rs @@ -109,6 +109,13 @@ impl DetectionTuner { self.snapshot } pub fn observe(&mut self, mode: DetectionMode, detected: bool, now: Instant) { + if self + .window + .as_ref() + .is_some_and(|window| window.mode != mode && !window.detected) + { + self.window = None; + } let window = self.window.get_or_insert(Observation { mode, began: now, @@ -218,6 +225,24 @@ mod tests { assert_eq!(tuner.snapshot().settings.interruption.value, 0.65); assert_eq!(tuner.snapshot().adjustments, 1); } + #[test] + fn idle_mode_changes_do_not_disable_interruption_learning() { + let mut tuner = tuner(); + let now = Instant::now(); + for n in 0..3 { + let time = now + Duration::from_secs(n * 40); + tuner.observe(DetectionMode::Listening, false, time); + tuner.observe( + DetectionMode::Interruption, + true, + time + Duration::from_secs(1), + ); + tuner.no_result(time + Duration::from_secs(7)); + } + assert_eq!(tuner.snapshot().settings.interruption.value, 0.66); + assert_eq!(tuner.snapshot().settings.listening.value, 0.5); + } + #[test] fn manual_mode_never_learns() { let settings = DetectionSettings::default(); diff --git a/src-tauri/crates/berd-call/src/detection_profiles.rs b/src-tauri/crates/berd-call/src/detection_profiles.rs index b48075143..ca62d04bb 100644 --- a/src-tauri/crates/berd-call/src/detection_profiles.rs +++ b/src-tauri/crates/berd-call/src/detection_profiles.rs @@ -85,6 +85,9 @@ fn write( if expected.is_some_and(|expected| profiles.profiles.get(&key(setup)).copied() != expected) { return Ok(false); } + if profiles.profiles.get(&key(setup)) == Some(&settings) { + return Ok(true); + } profiles.profiles.insert(key(setup), settings); if profiles.profiles.len() > 64 { return Err("too many speech profiles".into()); From 52e184bf2bcd4f7bf70901ab257a3540fc23f518 Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Thu, 22 Oct 2026 07:22:16 -0400 Subject: [PATCH 06/10] test: keep adaptive threshold time monotonic across resets Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/src/detection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/crates/berd-call/src/detection.rs b/src-tauri/crates/berd-call/src/detection.rs index 46effb229..45f73de62 100644 --- a/src-tauri/crates/berd-call/src/detection.rs +++ b/src-tauri/crates/berd-call/src/detection.rs @@ -282,7 +282,7 @@ mod tests { tuner.snapshot.settings.listening.value = 0.5; tuner.reset_evidence(); for n in 0..6 { - let time = now + Duration::from_secs(n); + let time = now + Duration::from_secs(1000 + n); tuner.observe(DetectionMode::Listening, true, time); tuner.no_result(time); } From 50e48a4476741759fde25b2ce1ac1059e5c0b17c Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Thu, 22 Oct 2026 07:25:30 -0400 Subject: [PATCH 07/10] berd-call: version the adaptive detection session messages Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/PROTOCOL.md | 13 +++++++++++-- src-tauri/crates/berd-call/src/protocol.rs | 6 +++--- src-tauri/crates/berd-call/src/session_framing.rs | 2 +- .../crates/berd-call/tests/session_protocol.rs | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index d38454cab..f31d2c301 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -58,12 +58,12 @@ initial policy; a host-specific `auto` mode must be resolved before the request. {"type":"hello","id":1,"input_during_tts":"allow_barge_in","status_sound_output_device":"MacBook Pro Speakers"} ``` -The response uses `protocol:5` as the exact session message-set version. The +The response uses `protocol:6` as the exact session message-set version. The parent must reject a version it does not support; the JSON version is independent of the fixed binary framing marker described below: ```json -{"type":"ready","id":1,"protocol":5,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} +{"type":"ready","id":1,"protocol":6,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} ``` The `session.tts` object is the authoritative, sanitized TTS configuration. @@ -397,3 +397,12 @@ receipts, flushes, and exits. EOF, malformed framing, fatal input failure, or process death cancels both authorities without transparent restart. A fatal error is flushed exactly once and followed by no protocol output. + + +## Local speech detection (protocol 6) + +Conventional session readiness includes an optional `session.detection` snapshot with `settings.listening` and `settings.interruption` (`value`, `dynamic`), plus `adjustments`, `no_result` and `missed_detection` counters. These local controls are separate from Realtime Spokesperson turn detection. + +A host sends `{"type":"set_detection_settings","id":42,"settings":{"listening":{"value":0.7,"dynamic":true},"interruption":{"value":0.8,"dynamic":false}}}` to replace both settings atomically and discard stale input/learning evidence. The session returns `detection_settings_result` with that ID and an applied `snapshot`, or `snapshot:null` and a `message` explaining rejection. Invalid settings leave the current values unchanged. Dynamic choices require macOS STT. + +During adaptive tuning, the session sends `detection_settings_result` with `id:null` and the new snapshot. Hosts must handle these unsolicited updates, report applied values, and keep profile persistence associated with their actual input/output device pair. Protocol 6 advances the exact message-set version because protocol 5 hosts cannot parse these new messages. diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index d489afc78..5155f1937 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -395,7 +395,7 @@ mod tests { #[test] fn host_decodes_realtime_ready_snapshot() { - let message = r#"{"type":"ready","id":1,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-realtime-2.1","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"#; + let message = r#"{"type":"ready","id":1,"protocol":6,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-realtime-2.1","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"#; let decoded = serde_json::from_str::(message).unwrap(); assert!(matches!(decoded, SessionMessage::Ready { id: 1, .. })); } @@ -421,7 +421,7 @@ mod tests { assert_eq!( serde_json::to_string(&SessionMessage::Ready { id: 4, - protocol: 5, + protocol: 6, session: VoiceSessionSnapshot { detection: None, tts: TtsConfigurationSnapshot { @@ -447,7 +447,7 @@ mod tests { }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"revision":1,"backend":"openai","model":"gpt-realtime","voice":"cedar","rate":1.0}}}"# + r#"{"type":"ready","id":4,"protocol":6,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"revision":1,"backend":"openai","model":"gpt-realtime","voice":"cedar","rate":1.0}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-call/src/session_framing.rs b/src-tauri/crates/berd-call/src/session_framing.rs index 5401ab4fa..8fedbf5d8 100644 --- a/src-tauri/crates/berd-call/src/session_framing.rs +++ b/src-tauri/crates/berd-call/src/session_framing.rs @@ -1,4 +1,4 @@ -pub(crate) const SESSION_PROTOCOL_VERSION: u32 = 5; +pub(crate) const SESSION_PROTOCOL_VERSION: u32 = 6; pub(crate) const FRAME_MAGIC: [u8; 2] = *b"BV"; pub(crate) const FRAME_MARKER: u8 = 3; pub(crate) const JSON_FRAME_KIND: u8 = 1; diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 391cd6ebf..b7d84a124 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -2904,7 +2904,7 @@ fn siri_session_reaches_ready_without_openai_credentials() { stdin.flush().unwrap(); let ready = receive(); assert_eq!(ready["type"], "ready"); - assert_eq!(ready["protocol"], 5); + assert_eq!(ready["protocol"], 6); assert_eq!(ready["session"]["tts"]["backend"], "siri"); assert_eq!(ready["session"]["tts"]["voice"], voice); assert_eq!(ready["session"]["tts"]["language"], language); From 45e15bba0d28a7f013d01b9cd08712c6dae51059 Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Thu, 22 Oct 2026 07:34:12 -0400 Subject: [PATCH 08/10] berd-call: preserve explicit speech settings over device profiles Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/README.md | 4 +- src-tauri/crates/berd-call/src/detection.rs | 67 +++++++++++ .../berd-call/src/detection_profiles.rs | 52 +++++++++ .../crates/berd-call/src/host_session.rs | 107 +++++++++++++++++- src-tauri/crates/berd-call/src/input.rs | 46 +++++++- .../crates/berd-call/src/saved_settings.rs | 9 +- 6 files changed, 272 insertions(+), 13 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 5b4ecaaaa..990dbb43e 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -15,7 +15,7 @@ Conventional calls accept independent `--listening-threshold 0..1` (default `0.5 On macOS, a local speech detection also marks recognition pending. After local speech ends, admission continues waiting for recognition to settle, with a five-second no-result fallback. Raising the listening threshold can reduce false starts in background noise; it does not shorten that fallback or prove a particular microphone's false detections are resolved. -Saved call arguments retain both values independently. To change a running call, supply its complete session configuration, for example `settings --restart --voice Aaron --language en-US --listening-threshold 0.8 --interruption-threshold 0.7`. Restart replaces the session options; include the existing backend, model, rate and other choices you want to retain. +Device profiles retain both values independently; legacy saved call arguments are accepted during migration. To change a running call, supply its complete session configuration, for example `settings --restart --voice Aaron --language en-US --listening-threshold 0.8 --interruption-threshold 0.7`. Restart replaces the session options; include the existing backend, model, rate and other choices you want to retain. ### Experimental dynamic tuning and audio profiles @@ -23,7 +23,7 @@ With macOS speech recognition, `--dynamic-listening true` and `--dynamic-interru A local detection that settles without a nonempty transcript suggests raising sensitivity's threshold. A nonempty transcript after at least five seconds without local detection suggests lowering it. Recognized speech clears false-trigger evidence. Split-result grace periods, mode changes, mute/reset epochs and device changes discard ambiguous evidence. These are experimental heuristics: recognition failure and playback leakage can still mislead them. They do not establish an optimal threshold or replace acoustic echo cancellation. -The standalone macOS host saves applied thresholds and dynamic choices in `detection-profiles.json` beside its saved settings, keyed by the stable input/output device UID pair. It restores the pair on call startup and checks for device changes once per second. A new pair starts with defaults during a call. Profile files contain settings, not audio recordings or transcript text. An explicit profile edit takes precedence over a late automatic update. +The standalone macOS host saves applied thresholds and dynamic choices in `detection-profiles.json` beside its saved settings, keyed by the stable input/output device UID pair. It restores the pair on call startup and checks for device changes once per second. Explicit threshold or dynamic flags override the matching profile choice; unspecified choices retain their saved values. Detection flags are removed from inherited session arguments so a later voice-only restart restores learned values. A new pair starts with defaults during a call. Profile files contain settings, not audio recordings or transcript text. An explicit profile edit takes precedence over a late automatic update. `berd-call detection-profile` reports the default audio setup and saved choices without starting capture. `berd-call detection-profile --settings '{"listening":{"value":0.70,"dynamic":true},"interruption":{"value":0.80,"dynamic":false}}'` saves choices for that pair. During a call, `berd-call status` exposes applied values and evidence counters in `session.detection`, plus `audioSetup`. Saving a profile alone affects its next restoration; the plugin restarts the call session when a user changes a control. Dynamic adjustments apply during the current session. diff --git a/src-tauri/crates/berd-call/src/detection.rs b/src-tauri/crates/berd-call/src/detection.rs index 45f73de62..20d836328 100644 --- a/src-tauri/crates/berd-call/src/detection.rs +++ b/src-tauri/crates/berd-call/src/detection.rs @@ -17,6 +17,43 @@ pub struct DetectionSettings { pub interruption: Threshold, } +pub fn is_detection_flag(flag: &str) -> bool { + matches!( + flag, + "--listening-threshold" + | "--interruption-threshold" + | "--dynamic-listening" + | "--dynamic-interruption" + ) +} + +pub fn without_detection_arguments(arguments: &[String]) -> Vec { + filter_arguments(arguments, false) +} + +pub fn detection_arguments(arguments: &[String]) -> Vec { + filter_arguments(arguments, true) +} + +fn filter_arguments(arguments: &[String], detection: bool) -> Vec { + let mut result = Vec::new(); + let mut index = 0; + while index < arguments.len() { + let flag = &arguments[index]; + let count = if flag == "session" || flag == "--allow-direct-agent-speech" { + 1 + } else { + 2 + }; + let end = (index + count).min(arguments.len()); + if is_detection_flag(flag) == detection { + result.extend_from_slice(&arguments[index..end]); + } + index = end; + } + result +} + impl Default for DetectionSettings { fn default() -> Self { Self { @@ -212,6 +249,36 @@ mod tests { }, }) } + #[test] + fn detection_flags_do_not_consume_flag_shaped_hint_values() { + let arguments = [ + "session", + "--stt-hint", + "--listening-threshold", + "--language", + "en-US", + "--listening-threshold", + "0.8", + "--allow-direct-agent-speech", + ] + .map(str::to_string); + assert_eq!( + detection_arguments(&arguments), + ["--listening-threshold", "0.8"] + ); + assert_eq!( + without_detection_arguments(&arguments), + [ + "session", + "--stt-hint", + "--listening-threshold", + "--language", + "en-US", + "--allow-direct-agent-speech" + ] + ); + } + #[test] fn repeated_no_result_moves_only_the_detecting_mode_one_tick() { let mut tuner = tuner(); diff --git a/src-tauri/crates/berd-call/src/detection_profiles.rs b/src-tauri/crates/berd-call/src/detection_profiles.rs index ca62d04bb..f7c5279ed 100644 --- a/src-tauri/crates/berd-call/src/detection_profiles.rs +++ b/src-tauri/crates/berd-call/src/detection_profiles.rs @@ -12,6 +12,28 @@ struct Profiles { profiles: BTreeMap, } +pub fn settings_for_start( + persisted: Option, + configured: DetectionSettings, + arguments: &[String], +) -> Result { + let mut settings = persisted.unwrap_or(configured); + for pair in berd_call::detection::detection_arguments(arguments).chunks_exact(2) { + match pair[0].as_str() { + "--listening-threshold" => settings.listening.value = configured.listening.value, + "--interruption-threshold" => { + settings.interruption.value = configured.interruption.value + } + "--dynamic-listening" => settings.listening.dynamic = configured.listening.dynamic, + "--dynamic-interruption" => { + settings.interruption.dynamic = configured.interruption.dynamic + } + _ => {} + } + } + settings.validate() +} + pub fn path() -> Result { Ok(crate::saved_settings::path()?.with_file_name("detection-profiles.json")) } @@ -115,6 +137,36 @@ mod tests { output_name: "same display name".into(), } } + #[test] + fn explicit_start_and_restart_flags_override_only_their_profile_choices() { + let mut profile = DetectionSettings::default(); + profile.listening.value = 0.73; + profile.listening.dynamic = true; + profile.interruption.value = 0.82; + let mut configured = DetectionSettings::default(); + configured.listening.value = 0.8; + configured.interruption.dynamic = true; + let arguments = [ + "session", + "--listening-threshold", + "0.8", + "--dynamic-interruption", + "true", + ] + .map(str::to_string); + let restored = settings_for_start(Some(profile), configured, &arguments).unwrap(); + assert_eq!(restored.listening.value, 0.8); + assert!(restored.listening.dynamic); + assert_eq!(restored.interruption.value, 0.82); + assert!(restored.interruption.dynamic); + let inherited = berd_call::detection::without_detection_arguments(&arguments); + assert_eq!(inherited, ["session"]); + assert_eq!( + settings_for_start(Some(profile), configured, &inherited).unwrap(), + profile + ); + } + #[test] fn late_learning_cannot_overwrite_an_explicit_profile_edit() { let dir = tempfile::tempdir().unwrap(); diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 210d8191f..43015bd5e 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -435,7 +435,7 @@ fn start_session( .store(muted.load(Ordering::SeqCst), Ordering::SeqCst); actor.capture_muted = muted; actor.capture_mute_epoch = mute_epoch; - actor.restore_detection_profile()?; + actor.restore_detection_profile(&arguments)?; actor.system_input_mute = system_input_mute; if actor.system_input_mute.is_none() { match crate::system_input_mute::SystemInputMute::install( @@ -485,7 +485,7 @@ fn start_session( || arguments .iter() .any(|argument| argument == crate::DIRECT_AGENT_SPEECH), - session_arguments: arguments, + session_arguments: berd_call::detection::without_detection_arguments(&arguments), }); Ok(StartedSession { child, @@ -1190,7 +1190,7 @@ impl SessionActor { } } - fn restore_detection_profile(&mut self) -> Result<(), String> { + fn restore_detection_profile(&mut self, arguments: &[String]) -> Result<(), String> { let detection = self .session .lock() @@ -1204,7 +1204,8 @@ impl SessionActor { }; let path = crate::detection_profiles::path()?; let persisted = crate::detection_profiles::load(&path, &setup)?; - let settings = persisted.unwrap_or(snapshot.settings); + let settings = + crate::detection_profiles::settings_for_start(persisted, snapshot.settings, arguments)?; let id = self.next_id(); send_request( &self.writer, @@ -1231,6 +1232,19 @@ impl SessionActor { let Some(setup) = berd_call::macos_audio_route::default_audio_setup() else { return Ok(()); }; + self.switch_detection_profile(setup) + } + + fn switch_detection_profile( + &mut self, + setup: berd_call::macos_audio_route::AudioSetup, + ) -> Result<(), String> { + let Some(profile) = self.detection_profile.as_mut() else { + return Ok(()); + }; + if profile.pending.is_some() { + return Ok(()); + } if setup.input_uid == profile.setup.input_uid && setup.output_uid == profile.setup.output_uid { @@ -3916,6 +3930,91 @@ mod tests { ) } + #[test] + fn device_switch_restores_the_saved_pair_and_ignores_old_learning_before_ack() { + let setup = |name: &str| berd_call::macos_audio_route::AudioSetup { + input_uid: format!("{name}-mic"), + output_uid: format!("{name}-output"), + input_name: name.into(), + output_name: name.into(), + }; + let first = setup("first"); + let second = setup("second"); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("profiles.json"); + let mut first_settings = berd_call::detection::DetectionSettings::default(); + first_settings.listening.value = 0.71; + let mut second_settings = first_settings; + second_settings.interruption.value = 0.84; + crate::detection_profiles::save(&path, &first, first_settings).unwrap(); + crate::detection_profiles::save(&path, &second, second_settings).unwrap(); + let mut child = Command::new("/bin/cat") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let writer = Arc::new(Mutex::new(child.stdin.take().unwrap())); + let (_event_tx, events) = mpsc::sync_channel(1); + let (_command_tx, commands) = mpsc::sync_channel(1); + let (audio, _audio_rx) = mpsc::sync_channel(1); + let mut actor = test_actor(writer, events, commands, audio); + actor.detection_profile = Some(ActiveDetectionProfile { + setup: first.clone(), + path, + persisted: Some(first_settings), + next_check: Instant::now(), + pending: None, + }); + actor.switch_detection_profile(second.clone()).unwrap(); + let id = actor.detection_profile.as_ref().unwrap().pending; + actor + .handle_event(SessionMessage::DetectionSettingsResult { + id: None, + snapshot: Some( + berd_call::detection::DetectionTuner::new(first_settings).snapshot(), + ), + message: None, + }) + .unwrap(); + assert!(actor.session.lock().unwrap().detection.is_none()); + actor + .handle_event(SessionMessage::DetectionSettingsResult { + id, + snapshot: Some( + berd_call::detection::DetectionTuner::new(second_settings).snapshot(), + ), + message: None, + }) + .unwrap(); + assert_eq!( + actor.session.lock().unwrap().detection.unwrap().settings, + second_settings + ); + actor.switch_detection_profile(first).unwrap(); + assert_eq!( + actor.detection_profile.as_ref().unwrap().persisted, + Some(first_settings) + ); + drop(actor); + let output = child.wait_with_output().unwrap().stdout; + let mut position = 0; + let mut settings = Vec::new(); + while position < output.len() { + let length = + u32::from_le_bytes(output[position + 4..position + 8].try_into().unwrap()) as usize; + let request: SessionRequest = + serde_json::from_slice(&output[position + 8..position + 8 + length]).unwrap(); + if let SessionRequest::SetDetectionSettings { + settings: applied, .. + } = request + { + settings.push(applied); + } + position += 8 + length; + } + assert_eq!(settings, [second_settings, first_settings]); + } + #[test] fn polling_waits_for_finalized_input_and_releases_waiters_on_shutdown() { let mut child = Command::new("/bin/cat") diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index 0d6e51410..2b678f81b 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -283,6 +283,11 @@ impl VoiceInputControls { #[cfg(target_os = "macos")] fn observe_detection(&self, detected: bool, epoch: u64) { + self.observe_detection_at(detected, epoch, Instant::now()); + } + + #[cfg(target_os = "macos")] + fn observe_detection_at(&self, detected: bool, epoch: u64, now: Instant) { let mode = if self.shared.assistant_speaking.load(Ordering::Acquire) { crate::detection::DetectionMode::Interruption } else { @@ -293,22 +298,27 @@ impl VoiceInputControls { return; } if let Some(tuner) = detection.as_mut() { - tuner.observe(mode, detected, Instant::now()); + tuner.observe(mode, detected, now); } } } #[cfg(target_os = "macos")] fn detection_result(&self, recognized: bool, epoch: u64) { + self.detection_result_at(recognized, epoch, Instant::now()); + } + + #[cfg(target_os = "macos")] + fn detection_result_at(&self, recognized: bool, epoch: u64, now: Instant) { if let Ok(mut detection) = self.shared.detection.lock() { if self.mute_epoch() != epoch { return; } if let Some(tuner) = detection.as_mut() { if recognized { - tuner.final_transcript(Instant::now()); + tuner.final_transcript(now); } else { - tuner.no_result(Instant::now()); + tuner.no_result(now); } } } @@ -3155,6 +3165,36 @@ mod tests { assert!(!finished.discard_on_shutdown.load(Ordering::Acquire)); } + #[cfg(target_os = "macos")] + #[test] + fn runtime_feedback_changes_only_the_active_threshold_and_applies_without_restart() { + let controls = VoiceInputControls::default(); + let mut settings = crate::detection::DetectionSettings::default(); + settings.listening.dynamic = true; + settings.interruption.dynamic = true; + controls.set_detection(settings).unwrap(); + let epoch = controls.mute_epoch(); + let now = Instant::now(); + for n in 0..3 { + let time = now + Duration::from_secs(n * 40); + controls.observe_detection_at(true, epoch, time); + controls.detection_result_at(false, epoch, time + Duration::from_secs(6)); + } + assert_eq!(controls.vad_threshold(0.5), 0.51); + let playback = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + for n in 0..3 { + let time = now + Duration::from_secs(120 + n * 40); + controls.observe_detection_at(false, epoch, time); + controls.detection_result_at(true, epoch, time + Duration::from_secs(6)); + } + assert_eq!(controls.vad_threshold(0.5), 0.64); + assert_eq!(controls.detection().unwrap().settings.listening.value, 0.51); + drop(playback); + assert_eq!(controls.vad_threshold(0.5), 0.51); + } + #[cfg(target_os = "macos")] #[test] fn stale_input_generation_cannot_train_a_new_profile() { diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index 9df0dfdfc..53ad6d841 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -143,10 +143,11 @@ pub(crate) fn persistable_arguments(arguments: &[String]) -> Vec { arguments .chunks_exact(2) .filter(|pair| { - !matches!( - pair[0].as_str(), - "--realtime-url" | "--stt-url" | "--tts-url" | "--stt-hint" - ) + !berd_call::detection::is_detection_flag(&pair[0]) + && !matches!( + pair[0].as_str(), + "--realtime-url" | "--stt-url" | "--tts-url" | "--stt-hint" + ) }) .flatten() .cloned() From b67aaacdedd547bff513180b566a9ea4384c62dd Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Thu, 22 Oct 2026 08:26:21 -0400 Subject: [PATCH 09/10] Adjust dynamic VAD on each qualifying outcome Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/README.md | 2 +- src-tauri/crates/berd-call/VAD_VISION.md | 2 +- src-tauri/crates/berd-call/src/detection.rs | 70 +++++---------------- src-tauri/crates/berd-call/src/input.rs | 4 +- 4 files changed, 18 insertions(+), 60 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 990dbb43e..2a5da9aa1 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -19,7 +19,7 @@ Device profiles retain both values independently; legacy saved call arguments ar ### Experimental dynamic tuning and audio profiles -With macOS speech recognition, `--dynamic-listening true` and `--dynamic-interruption true` independently opt into outcome-based tuning. Both default to false. Each enabled threshold moves one 0.01 tick after three unmatched observations, at most once every 30 seconds. Dynamic values start and remain within 0.10 to 0.95; manual thresholds still accept 0 to 1. +With macOS speech recognition, `--dynamic-listening true` and `--dynamic-interruption true` independently opt into outcome-based tuning. Both default to false. Each enabled threshold moves one 0.01 tick after each qualifying unmatched outcome, without a cooldown. Dynamic values start and remain within 0.10 to 0.95; manual thresholds still accept 0 to 1. A local detection that settles without a nonempty transcript suggests raising sensitivity's threshold. A nonempty transcript after at least five seconds without local detection suggests lowering it. Recognized speech clears false-trigger evidence. Split-result grace periods, mode changes, mute/reset epochs and device changes discard ambiguous evidence. These are experimental heuristics: recognition failure and playback leakage can still mislead them. They do not establish an optimal threshold or replace acoustic echo cancellation. diff --git a/src-tauri/crates/berd-call/VAD_VISION.md b/src-tauri/crates/berd-call/VAD_VISION.md index d8e810f53..216a8aca9 100644 --- a/src-tauri/crates/berd-call/VAD_VISION.md +++ b/src-tauri/crates/berd-call/VAD_VISION.md @@ -4,7 +4,7 @@ Proposed direction, October 22, 2026. Independent static thresholds are implemen A caller should be able to stop background noise holding replies without losing quiet speech. Listening and interruption each have a slider, visible current value, and an opt-in Dynamically adjust checkbox. Adjustments move one 0.01 tick at a time within 0.10 to 0.95. Manual values remain available across the full 0 to 1 range. -For the initial experimental macOS recognizer, repeated local detections that settle without a nonempty transcript are evidence to increase the threshold. Nonempty transcripts without corresponding local detection provide evidence to decrease it. These are heuristics, not ground truth: recognizer failures, late or split results and playback leakage can mislead them. Ignore ambiguous observations across mute/reset, playback-mode and device changes; rate-limit changes and expose evidence counters with current values. Other recognition backends require their own reliable outcome signals before enabling adaptive tuning. +For the initial experimental macOS recognizer, local detections that settle without a nonempty transcript are evidence to increase the threshold. Nonempty transcripts without corresponding local detection provide evidence to decrease it. These are heuristics, not ground truth: recognizer failures, late or split results and playback leakage can mislead them. Ignore ambiguous observations across mute/reset, playback-mode and device changes; move one tick per qualifying outcome and expose evidence counters with current values. Other recognition backends require their own reliable outcome signals before enabling adaptive tuning. Save independent values and dynamic choices by stable input/output device identity pair. AirPods input/output, another headset, and the built-in input/output pair retain separate profiles. Restoring a profile must discard old learning evidence so one setup cannot train another. No microphone recordings or transcript text are stored in these profiles. diff --git a/src-tauri/crates/berd-call/src/detection.rs b/src-tauri/crates/berd-call/src/detection.rs index 20d836328..c6e51e56a 100644 --- a/src-tauri/crates/berd-call/src/detection.rs +++ b/src-tauri/crates/berd-call/src/detection.rs @@ -98,14 +98,12 @@ pub enum DetectionMode { } const STEP: f32 = 0.01; -const COOLDOWN: Duration = Duration::from_secs(30); const RESULT_GRACE: Duration = Duration::from_secs(5); pub struct DetectionTuner { snapshot: DetectionSnapshot, window: Option, grace_until: Option, - evidence: [Evidence; 2], } struct Observation { @@ -115,19 +113,6 @@ struct Observation { ambiguous: bool, } -#[derive(Default)] -struct Evidence { - no_result: u8, - missed: u8, - last_adjustment: Option, -} - -impl DetectionMode { - fn index(self) -> usize { - usize::from(self == Self::Interruption) - } -} - impl DetectionTuner { pub fn new(settings: DetectionSettings) -> Self { Self { @@ -139,7 +124,6 @@ impl DetectionTuner { }, window: None, grace_until: None, - evidence: Default::default(), } } pub fn snapshot(&self) -> DetectionSnapshot { @@ -164,47 +148,27 @@ impl DetectionTuner { } pub fn final_transcript(&mut self, now: Instant) { if let Some(window) = self.window.take().filter(|window| !window.ambiguous) { - let evidence = &mut self.evidence[window.mode.index()]; - evidence.no_result = 0; - if window.detected { - evidence.missed = 0; - } else if now.duration_since(window.began) >= RESULT_GRACE + if !window.detected + && now.duration_since(window.began) >= RESULT_GRACE && self.grace_until.is_none_or(|until| now >= until) { self.snapshot.missed_detection += 1; - evidence.missed = evidence.missed.saturating_add(1); - self.adjust(window.mode, false, now); + self.adjust(window.mode, false); } } self.grace_until = Some(now + RESULT_GRACE); } - pub fn no_result(&mut self, now: Instant) { + pub fn no_result(&mut self, _now: Instant) { if let Some(window) = self .window .take() .filter(|window| window.detected && !window.ambiguous) { self.snapshot.no_result += 1; - let evidence = &mut self.evidence[window.mode.index()]; - evidence.missed = 0; - evidence.no_result = evidence.no_result.saturating_add(1); - self.adjust(window.mode, true, now); + self.adjust(window.mode, true); } } - fn adjust(&mut self, mode: DetectionMode, increase: bool, now: Instant) { - let evidence = &mut self.evidence[mode.index()]; - let count = if increase { - evidence.no_result - } else { - evidence.missed - }; - if count < 3 - || evidence - .last_adjustment - .is_some_and(|last| now.duration_since(last) < COOLDOWN) - { - return; - } + fn adjust(&mut self, mode: DetectionMode, increase: bool) { let threshold = match mode { DetectionMode::Listening => &mut self.snapshot.settings.listening, DetectionMode::Interruption => &mut self.snapshot.settings.interruption, @@ -220,17 +184,10 @@ impl DetectionTuner { } threshold.value = next; self.snapshot.adjustments += 1; - evidence.last_adjustment = Some(now); - evidence.no_result = 0; - evidence.missed = 0; } pub fn reset_evidence(&mut self) { self.window = None; self.grace_until = None; - for evidence in &mut self.evidence { - evidence.no_result = 0; - evidence.missed = 0; - } } } @@ -280,7 +237,7 @@ mod tests { } #[test] - fn repeated_no_result_moves_only_the_detecting_mode_one_tick() { + fn each_no_result_moves_only_the_detecting_mode_one_tick() { let mut tuner = tuner(); let now = Instant::now(); for n in 0..3 { @@ -288,9 +245,9 @@ mod tests { tuner.observe(DetectionMode::Listening, true, time); tuner.no_result(time + Duration::from_secs(6)); } - assert_eq!(tuner.snapshot().settings.listening.value, 0.51); + assert_eq!(tuner.snapshot().settings.listening.value, 0.53); assert_eq!(tuner.snapshot().settings.interruption.value, 0.65); - assert_eq!(tuner.snapshot().adjustments, 1); + assert_eq!(tuner.snapshot().adjustments, 3); } #[test] fn idle_mode_changes_do_not_disable_interruption_learning() { @@ -306,7 +263,7 @@ mod tests { ); tuner.no_result(time + Duration::from_secs(7)); } - assert_eq!(tuner.snapshot().settings.interruption.value, 0.66); + assert_eq!(tuner.snapshot().settings.interruption.value, 0.68); assert_eq!(tuner.snapshot().settings.listening.value, 0.5); } @@ -331,11 +288,11 @@ mod tests { tuner.observe(DetectionMode::Interruption, false, time); tuner.final_transcript(time + Duration::from_secs(6)); } - assert_eq!(tuner.snapshot().settings.interruption.value, 0.64); + assert_eq!(tuner.snapshot().settings.interruption.value, 0.62); assert_eq!(tuner.snapshot().settings.listening.value, 0.5); } #[test] - fn adjustments_are_rate_limited_and_stay_inside_dynamic_bounds() { + fn adjustments_have_no_cooldown_and_stay_inside_dynamic_bounds() { let mut tuner = tuner(); tuner.snapshot.settings.listening.value = 0.94; let now = Instant::now(); @@ -353,7 +310,8 @@ mod tests { tuner.observe(DetectionMode::Listening, true, time); tuner.no_result(time); } - assert_eq!(tuner.snapshot().adjustments, 2); + assert_eq!(tuner.snapshot().adjustments, 7); + assert_eq!(tuner.snapshot().settings.listening.value, 0.56); } #[test] diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index 2b678f81b..52b312197 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -3175,7 +3175,7 @@ mod tests { controls.set_detection(settings).unwrap(); let epoch = controls.mute_epoch(); let now = Instant::now(); - for n in 0..3 { + for n in 0..1 { let time = now + Duration::from_secs(n * 40); controls.observe_detection_at(true, epoch, time); controls.detection_result_at(false, epoch, time + Duration::from_secs(6)); @@ -3184,7 +3184,7 @@ mod tests { let playback = controls .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) .unwrap(); - for n in 0..3 { + for n in 0..1 { let time = now + Duration::from_secs(120 + n * 40); controls.observe_detection_at(false, epoch, time); controls.detection_result_at(true, epoch, time + Duration::from_secs(6)); From 6d889259cb1683af7111e445c36e99d989dcb419 Mon Sep 17 00:00:00 2001 From: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> Date: Thu, 22 Oct 2026 08:30:36 -0400 Subject: [PATCH 10/10] Expose configurable Apple VAD onset and release timing Signed-off-by: Zack <04f47b290b155b3b8b782b11501cdb15f1e21e82dfceefd01ff0ed405459d927@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-call/PROTOCOL.md | 8 +- src-tauri/crates/berd-call/README.md | 3 + src-tauri/crates/berd-call/src/detection.rs | 88 +++++++++++++++++++ src-tauri/crates/berd-call/src/input.rs | 31 ++++--- src-tauri/crates/berd-call/src/main.rs | 1 + src-tauri/crates/berd-call/src/protocol.rs | 6 +- .../crates/berd-call/src/session_framing.rs | 2 +- .../berd-call/tests/session_protocol.rs | 2 +- 8 files changed, 116 insertions(+), 25 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index f31d2c301..2f307a063 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -58,12 +58,12 @@ initial policy; a host-specific `auto` mode must be resolved before the request. {"type":"hello","id":1,"input_during_tts":"allow_barge_in","status_sound_output_device":"MacBook Pro Speakers"} ``` -The response uses `protocol:6` as the exact session message-set version. The +The response uses `protocol:7` as the exact session message-set version. The parent must reject a version it does not support; the JSON version is independent of the fixed binary framing marker described below: ```json -{"type":"ready","id":1,"protocol":6,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} +{"type":"ready","id":1,"protocol":7,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} ``` The `session.tts` object is the authoritative, sanitized TTS configuration. @@ -399,10 +399,10 @@ restart. A fatal error is flushed exactly once and followed by no protocol output. -## Local speech detection (protocol 6) +## Local speech detection (protocol 7) Conventional session readiness includes an optional `session.detection` snapshot with `settings.listening` and `settings.interruption` (`value`, `dynamic`), plus `adjustments`, `no_result` and `missed_detection` counters. These local controls are separate from Realtime Spokesperson turn detection. A host sends `{"type":"set_detection_settings","id":42,"settings":{"listening":{"value":0.7,"dynamic":true},"interruption":{"value":0.8,"dynamic":false}}}` to replace both settings atomically and discard stale input/learning evidence. The session returns `detection_settings_result` with that ID and an applied `snapshot`, or `snapshot:null` and a `message` explaining rejection. Invalid settings leave the current values unchanged. Dynamic choices require macOS STT. -During adaptive tuning, the session sends `detection_settings_result` with `id:null` and the new snapshot. Hosts must handle these unsolicited updates, report applied values, and keep profile persistence associated with their actual input/output device pair. Protocol 6 advances the exact message-set version because protocol 5 hosts cannot parse these new messages. +During adaptive tuning, the session sends `detection_settings_result` with `id:null` and the new snapshot. Hosts must handle these unsolicited updates, report applied values, and keep profile persistence associated with their actual input/output device pair. Protocol 7 adds `settings.timing` (`onset_ms`, `release_ms`) for Apple local VAD. Older profiles omit this field and receive defaults of 16 ms onset and 300 ms release. Final transcripts are emitted immediately, independently of the local activity gate. diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 2a5da9aa1..9762af1fc 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -21,6 +21,9 @@ Device profiles retain both values independently; legacy saved call arguments ar With macOS speech recognition, `--dynamic-listening true` and `--dynamic-interruption true` independently opt into outcome-based tuning. Both default to false. Each enabled threshold moves one 0.01 tick after each qualifying unmatched outcome, without a cooldown. Dynamic values start and remain within 0.10 to 0.95; manual thresholds still accept 0 to 1. +Apple recognition also supports per-audio-setup `timing` settings through `detection-profile --settings`: `onset_ms` (16 to 1000, default 16) and `release_ms` (16 to 3000, default 300). Timings round up to 16 ms audio frames and apply only to local activity detection. Finalized transcripts are forwarded immediately, independently of release timing. The recognizer continues to receive all microphone audio during onset gating. Other recognition backends retain their existing segmentation timing. + + A local detection that settles without a nonempty transcript suggests raising sensitivity's threshold. A nonempty transcript after at least five seconds without local detection suggests lowering it. Recognized speech clears false-trigger evidence. Split-result grace periods, mode changes, mute/reset epochs and device changes discard ambiguous evidence. These are experimental heuristics: recognition failure and playback leakage can still mislead them. They do not establish an optimal threshold or replace acoustic echo cancellation. The standalone macOS host saves applied thresholds and dynamic choices in `detection-profiles.json` beside its saved settings, keyed by the stable input/output device UID pair. It restores the pair on call startup and checks for device changes once per second. Explicit threshold or dynamic flags override the matching profile choice; unspecified choices retain their saved values. Detection flags are removed from inherited session arguments so a later voice-only restart restores learned values. A new pair starts with defaults during a call. Profile files contain settings, not audio recordings or transcript text. An explicit profile edit takes precedence over a late automatic update. diff --git a/src-tauri/crates/berd-call/src/detection.rs b/src-tauri/crates/berd-call/src/detection.rs index c6e51e56a..c1c8106b0 100644 --- a/src-tauri/crates/berd-call/src/detection.rs +++ b/src-tauri/crates/berd-call/src/detection.rs @@ -15,6 +15,51 @@ pub struct Threshold { pub struct DetectionSettings { pub listening: Threshold, pub interruption: Threshold, + #[serde(default)] + pub timing: VadTiming, +} + +/// Apple recognition keeps receiving audio while these local activity gates settle. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct VadTiming { + pub onset_ms: u32, + pub release_ms: u32, +} + +impl Default for VadTiming { + fn default() -> Self { + Self { + onset_ms: 16, + release_ms: 300, + } + } +} + +#[derive(Default)] +pub struct SpeechGate { + active: bool, + speech_frames: u32, + silence_frames: u32, +} + +impl SpeechGate { + pub fn observe(&mut self, detected: bool, timing: VadTiming) -> bool { + if detected { + self.silence_frames = 0; + self.speech_frames = self.speech_frames.saturating_add(1); + if self.speech_frames >= timing.onset_ms.div_ceil(16).max(1) { + self.active = true; + } + } else { + self.speech_frames = 0; + self.silence_frames = self.silence_frames.saturating_add(1); + if self.silence_frames >= timing.release_ms.div_ceil(16).max(1) { + self.active = false; + } + } + self.active + } } pub fn is_detection_flag(flag: &str) -> bool { @@ -57,6 +102,7 @@ fn filter_arguments(arguments: &[String], detection: bool) -> Vec { impl Default for DetectionSettings { fn default() -> Self { Self { + timing: VadTiming::default(), listening: Threshold { value: 0.5, dynamic: false, @@ -71,6 +117,11 @@ impl Default for DetectionSettings { impl DetectionSettings { pub fn validate(self) -> Result { + if !(16..=1000).contains(&self.timing.onset_ms) + || !(16..=3000).contains(&self.timing.release_ms) + { + return Err("VAD onset must be 16 to 1000 ms and release 16 to 3000 ms".into()); + } for threshold in [self.listening, self.interruption] { if !threshold.value.is_finite() || !(0.0..=1.0).contains(&threshold.value) { return Err("speech thresholds must be finite and between 0 and 1".into()); @@ -194,8 +245,45 @@ impl DetectionTuner { #[cfg(test)] mod tests { use super::*; + #[test] + fn gate_requires_consecutive_onset_and_release_frames() { + let mut gate = SpeechGate::default(); + let timing = VadTiming { + onset_ms: 32, + release_ms: 48, + }; + assert!(!gate.observe(true, timing)); + assert!(!gate.observe(false, timing)); + assert!(!gate.observe(true, timing)); + assert!(gate.observe(true, timing)); + assert!(gate.observe(false, timing)); + assert!(gate.observe(false, timing)); + assert!(!gate.observe(false, timing)); + } + + #[test] + fn legacy_profiles_get_default_timing_and_invalid_timing_is_rejected() { + let settings: DetectionSettings = + serde_json::from_str(r#"{"listening":{"value":0.5},"interruption":{"value":0.65}}"#) + .unwrap(); + assert_eq!(settings.timing, VadTiming::default()); + for timing in [ + VadTiming { + onset_ms: 0, + release_ms: 300, + }, + VadTiming { + onset_ms: 16, + release_ms: 3001, + }, + ] { + assert!(DetectionSettings { timing, ..settings }.validate().is_err()); + } + } + fn tuner() -> DetectionTuner { DetectionTuner::new(DetectionSettings { + timing: VadTiming::default(), listening: Threshold { value: 0.5, dynamic: true, diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index 52b312197..777cd3afd 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -1515,7 +1515,7 @@ fn mac_speech_worker( let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); let mut input_48k = Vec::new(); let mut leftover_16k = Vec::new(); - let mut silence_frames = 0; + let mut gate = crate::detection::SpeechGate::default(); let mut in_speech = false; let mut observed_epoch = controls.mute_epoch(); let mut pending = PendingRecognitions::new(); @@ -1548,7 +1548,7 @@ fn mac_speech_worker( settle_deadline = None; input_48k.clear(); leftover_16k.clear(); - silence_frames = 0; + gate = crate::detection::SpeechGate::default(); vad = earshot::Detector::new(earshot::DefaultPredictor::new()); match new_mac_recognizer(&hints) { Ok((next, next_events)) => { @@ -1578,7 +1578,7 @@ fn mac_speech_worker( } input_48k.clear(); leftover_16k.clear(); - silence_frames = 0; + gate = crate::detection::SpeechGate::default(); if std::mem::take(&mut in_speech) { let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); } @@ -1626,24 +1626,22 @@ fn mac_speech_worker( let clamped = clamp_vad_frame(&frame); let detected = vad.predict_f32(&clamped) > controls.vad_threshold(speech_vad_threshold); - controls.observe_detection(detected, observed_epoch); - if detected { - silence_frames = 0; - if !in_speech { - in_speech = true; + let timing = controls + .detection() + .map(|snapshot| snapshot.settings.timing) + .unwrap_or_default(); + let active = gate.observe(detected, timing); + controls.observe_detection(active && detected, observed_epoch); + if active != in_speech { + in_speech = active; + if active { if begin_mac_turn(&mut pending, &events, &mut settle_deadline).is_err() { return; } - let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(true)); - } - } else if in_speech { - silence_frames += 1; - if silence_frames >= SILENCE_FLUSH_FRAMES { - silence_frames = 0; - in_speech = false; + } else { end_mac_turn(&pending, &mut settle_deadline, Instant::now()); - let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); } + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(active)); } } } @@ -3217,6 +3215,7 @@ mod tests { fn applied_detection_settings_survive_playback_guards_and_reject_invalid_updates() { let controls = VoiceInputControls::default(); let settings = crate::detection::DetectionSettings { + timing: Default::default(), listening: crate::detection::Threshold { value: 0.81, dynamic: true, diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 634734683..a104ffb08 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -2348,6 +2348,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String config.endpoints.stt.as_deref(), &config.stt_hints, berd_call::detection::DetectionSettings { + timing: Default::default(), listening: berd_call::detection::Threshold { value: config.listening_threshold, dynamic: config.dynamic_listening, diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index 5155f1937..440777087 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -395,7 +395,7 @@ mod tests { #[test] fn host_decodes_realtime_ready_snapshot() { - let message = r#"{"type":"ready","id":1,"protocol":6,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-realtime-2.1","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"#; + let message = r#"{"type":"ready","id":1,"protocol":7,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-realtime-2.1","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"#; let decoded = serde_json::from_str::(message).unwrap(); assert!(matches!(decoded, SessionMessage::Ready { id: 1, .. })); } @@ -421,7 +421,7 @@ mod tests { assert_eq!( serde_json::to_string(&SessionMessage::Ready { id: 4, - protocol: 6, + protocol: 7, session: VoiceSessionSnapshot { detection: None, tts: TtsConfigurationSnapshot { @@ -447,7 +447,7 @@ mod tests { }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":6,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"revision":1,"backend":"openai","model":"gpt-realtime","voice":"cedar","rate":1.0}}}"# + r#"{"type":"ready","id":4,"protocol":7,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"revision":1,"backend":"openai","model":"gpt-realtime","voice":"cedar","rate":1.0}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-call/src/session_framing.rs b/src-tauri/crates/berd-call/src/session_framing.rs index 8fedbf5d8..94fa6aa49 100644 --- a/src-tauri/crates/berd-call/src/session_framing.rs +++ b/src-tauri/crates/berd-call/src/session_framing.rs @@ -1,4 +1,4 @@ -pub(crate) const SESSION_PROTOCOL_VERSION: u32 = 6; +pub(crate) const SESSION_PROTOCOL_VERSION: u32 = 7; pub(crate) const FRAME_MAGIC: [u8; 2] = *b"BV"; pub(crate) const FRAME_MARKER: u8 = 3; pub(crate) const JSON_FRAME_KIND: u8 = 1; diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index b7d84a124..d220432b3 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -2904,7 +2904,7 @@ fn siri_session_reaches_ready_without_openai_credentials() { stdin.flush().unwrap(); let ready = receive(); assert_eq!(ready["type"], "ready"); - assert_eq!(ready["protocol"], 6); + assert_eq!(ready["protocol"], 7); assert_eq!(ready["session"]["tts"]["backend"], "siri"); assert_eq!(ready["session"]["tts"]["voice"], voice); assert_eq!(ready["session"]["tts"]["language"], language);