From 320479584cd884a67ba08d66abcb88ec4bc450f8 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Sat, 31 Oct 2026 03:35:04 +0800 Subject: [PATCH 01/19] Keep trace capture responsive during daemon recovery --- bt-daemon/src/server/ingress.rs | 204 ++++++++++++++++- bt-daemon/src/server/maintenance.rs | 334 +++++++++++++++++++++++++++- bt-daemon/src/server/mod.rs | 74 ++++-- bt-daemon/src/server/recovery.rs | 32 ++- bt-daemon/src/server/sessions.rs | 4 + bt-daemon/src/transport.rs | 109 ++++++++- 6 files changed, 715 insertions(+), 42 deletions(-) diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index d2ae453d..47615a32 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -3,6 +3,169 @@ //! the journals as overflow when the queue is full. use super::*; +use std::collections::VecDeque; + +/// Live startup captures remain ordered and bounded. Evicted entries are +/// already durable; their source journal's high-water mark is replayed first. +pub(super) struct StartupBuffer { + pub(super) events: VecDeque, + pub(super) evicted: HashMap, +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct EvictedRange { + pub(super) from: u64, + pub(super) through: u64, +} + +impl StartupBuffer { + pub(super) fn new() -> Self { + Self { + events: VecDeque::new(), + evicted: HashMap::new(), + } + } + + fn push(&mut self, data_dir: &std::path::Path, event: PendingEvent) { + if self.events.len() == INGRESS_QUEUE_CAPACITY { + if let Some(oldest) = self.events.pop_front() { + let path = journal::source_journal_path( + data_dir, + &oldest.env.source, + &oldest.env.session_id, + ); + self.evicted + .entry(path) + .and_modify(|range| range.through = range.through.max(oldest.journal_through)) + .or_insert(EvictedRange { + from: oldest.replay_through, + through: oldest.journal_through, + }); + } + } + self.events.push_back(event); + } +} + +pub(super) enum StartupWork { + Evicted(HashMap), + Event(Box), + Finished, +} + +impl Daemon { + pub(super) fn next_startup_work(&self) -> StartupWork { + let mut state = self.startup_buffer.lock(); + let Some(buffer) = state.as_mut() else { + return StartupWork::Finished; + }; + if !buffer.evicted.is_empty() { + return StartupWork::Evicted(std::mem::take(&mut buffer.evicted)); + } + if let Some(event) = buffer.events.pop_front() { + return StartupWork::Event(Box::new(event)); + } + *state = None; + StartupWork::Finished + } +} + +pub(super) async fn drain_startup_buffer(daemon: &Arc) -> anyhow::Result<()> { + let mut cursors = HashMap::new(); + loop { + match daemon.next_startup_work() { + StartupWork::Evicted(bounds) => { + replay_evicted_startup(daemon, &mut cursors, &bounds).await? + } + StartupWork::Event(event) => dispatch_ingress_event(daemon, *event).await, + StartupWork::Finished => return Ok(()), + } + } +} + +pub(super) struct StartupCursor { + reader: journal::JournalReader, + position: u64, +} + +/// Advance each evicted journal range from its previous position, retaining +/// the reader's Pi delta context between advances. Older records are decoded +/// only to initialize that context; they are not dispatched again. +pub(super) async fn replay_evicted_startup( + daemon: &Arc, + cursors: &mut HashMap, + bounds: &HashMap, +) -> anyhow::Result<()> { + for (path, range) in bounds { + anyhow::ensure!( + journal::JournalReader::recorded_len(path).await >= range.through, + "evicted startup journal is truncated" + ); + if !cursors.contains_key(path) { + let reader = journal::JournalReader::open(path, range.from) + .await? + .ok_or_else(|| anyhow::anyhow!("evicted startup journal disappeared"))?; + cursors.insert( + path.clone(), + StartupCursor { + reader, + position: 0, + }, + ); + } + let cursor = cursors + .get_mut(path) + .ok_or_else(|| anyhow::anyhow!("startup cursor missing"))?; + anyhow::ensure!( + cursor.position <= range.from, + "evicted startup ranges overlap" + ); + cursor.reader.extend(range.from); + while cursor.position < range.from { + let entry = + cursor.reader.next_record().await?.ok_or_else(|| { + anyhow::anyhow!("evicted startup range begins inside a record") + })?; + cursor.position = entry.through; + } + anyhow::ensure!( + cursor.position == range.from, + "invalid evicted startup boundary" + ); + cursor.reader.extend(range.through); + while cursor.position < range.through { + let before = cursor.position; + let entry = cursor + .reader + .next_record() + .await? + .ok_or_else(|| anyhow::anyhow!("evicted startup range ends inside a record"))?; + cursor.position = entry.through; + if let journal::JournalRecord::Event(redacted) = entry.record { + let mut env = journal::envelope_from_redacted(redacted); + let Some(source) = daemon.translators.canonical_source(&env.source) else { + continue; + }; + env.source = source.to_owned(); + dispatch_ingress_event( + daemon, + PendingEvent { + env, + replay_through: before, + journal_through: entry.through, + settings_through: None, + }, + ) + .await; + } + } + anyhow::ensure!( + cursor.position == range.through, + "invalid evicted startup end" + ); + } + Ok(()) +} pub(super) enum IngressMsg { Event(Box), @@ -14,8 +177,19 @@ pub(super) enum IngressMsg { /// journals become the overflow queue and the worker catches them up in place. pub(super) const INGRESS_QUEUE_CAPACITY: usize = 64; -pub(super) fn spawn_ingress_worker(daemon: Arc, mut rx: mpsc::Receiver) { +pub(super) fn spawn_ingress_worker( + daemon: Arc, + mut rx: mpsc::Receiver, + startup: Option>, +) { tokio::spawn(async move { + if let Some(startup) = startup { + if startup.await.is_err() { + tracing::error!("startup recovery failed; stopping ingress"); + daemon.trigger_shutdown(); + return; + } + } loop { if daemon.ingress_overflow.swap(false, Ordering::AcqRel) { recover_ingress_overflow(&daemon).await; @@ -63,6 +237,10 @@ pub(super) async fn dispatch_ingress_event(daemon: &Arc, event: PendingE return; } + // Startup discovery rebuilds these observations before this worker is + // released, so a historical event cannot overwrite a newer capture. + daemon.remember_claude_transcripts(&event.env); + let journal_through = event.journal_through; // A child may arrive immediately after a parent's tool hook. Catch prior // session actors up before resolving a new session, entirely in the daemon. @@ -323,15 +501,21 @@ impl Daemon { .append_to_journal(&mut env) .await .map_err(|error| format!("journal failed: {error}"))?; - self.remember_claude_transcripts(&env); - match self - .ingress_tx - .try_send(IngressMsg::Event(Box::new(PendingEvent { - env, - replay_through, - journal_through, - settings_through: None, - }))) { + let event = PendingEvent { + env, + replay_through, + journal_through, + settings_through: None, + }; + let event = { + let mut state = self.startup_buffer.lock(); + if let Some(buffer) = state.as_mut() { + buffer.push(&self.data_dir, event); + return Ok(true); + } + event + }; + match self.ingress_tx.try_send(IngressMsg::Event(Box::new(event))) { Ok(()) => {} Err(mpsc::error::TrySendError::Full(_)) => { self.ingress_overflow.store(true, Ordering::Release); diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 61e9d45c..16107ef2 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -70,6 +70,17 @@ async fn collect_partitions(daemon: &Daemon, age: Duration) { if tokio::fs::try_exists(&wal).await.unwrap_or(true) && !old(&wal, age).await { continue; } + // Scanning a large journal must not hold the capture lock. The length + // and route set are revalidated under that lock before deletion. + let Ok(partition) = crate::pipeline::journal::SpanJournal::open(&storage).await else { + continue; + }; + let Ok(routes) = partition.routes().await else { + continue; + }; + if !captured_routes_registered(&daemon.data_dir, &path, recorded, &routes).await { + continue; + } let lock = daemon.session_lock(&first.source, &first.session_id); let _guard = lock.lock().await; let namespace = crate::ids::session_namespace(&first.source, &first.session_id); @@ -81,19 +92,24 @@ async fn collect_partitions(daemon: &Daemon, age: Duration) { if journal::JournalReader::recorded_len(&path).await != recorded { continue; } - let Ok(partition) = crate::pipeline::journal::SpanJournal::open(&storage).await else { + let Ok(current_routes) = partition.routes().await else { continue; }; - let Ok(routes) = partition.routes().await else { + let (Ok(current_routes), Ok(scanned_routes)) = ( + serde_json::to_value(¤t_routes), + serde_json::to_value(&routes), + ) else { continue; }; + if current_routes != scanned_routes { + continue; + } let pending = daemon.pending_sessions.lock().values().any(|state| { state.events.iter().any(|event| { event.env.source == first.source && event.env.session_id == first.session_id }) }); - if pending || !captured_routes_registered(&daemon.data_dir, &path, recorded, &routes).await - { + if pending { continue; } @@ -421,6 +437,316 @@ mod tests { source_version: None, } } + + #[tokio::test] + async fn capture_is_durable_while_startup_dispatch_is_blocked() { + let tmp = tempfile::tempdir().unwrap(); + let (ready, startup) = oneshot::channel(); + let daemon = Daemon::new_with_startup( + debug_serve_options("test", tmp.path()), + tmp.path().into(), + Some(startup), + ); + let env = event("during-startup", SessionRoute::default()); + assert!( + tokio::time::timeout(Duration::from_secs(1), daemon.capture_event(env.clone())) + .await + .unwrap() + .unwrap() + ); + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + assert!(tokio::fs::read_to_string(path) + .await + .unwrap() + .contains("during-startup")); + assert!(daemon.translations.lock().is_empty()); + drain_startup_buffer(&daemon).await.unwrap(); + ready.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), daemon.ingress_barrier()) + .await + .unwrap(); + assert!(!daemon.translations.lock().is_empty()); + drain_all(&daemon).await; + } + + #[tokio::test] + async fn startup_recovery_dispatches_old_session_before_new_capture() { + let tmp = tempfile::tempdir().unwrap(); + let env = event("resuming", SessionRoute::default()); + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let old_end = writer.append(&env).await.unwrap(); + drop(writer); + let (ready, startup) = oneshot::channel(); + let daemon = Daemon::new_with_startup( + debug_serve_options("test", tmp.path()), + tmp.path().into(), + Some(startup), + ); + let mut next = env.clone(); + next.event = "next".into(); + next.ts_ms = 2; + assert!(daemon.capture_event(next).await.unwrap()); + let recovered = discover_unprocessed_journals(&daemon).await; + assert_eq!(recovered.len(), 1); + assert_eq!(recovered[0].journal_through, old_end); + for event in recovered { + dispatch_ingress_event(&daemon, event).await; + } + drain_startup_buffer(&daemon).await.unwrap(); + ready.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), daemon.ingress_barrier()) + .await + .unwrap(); + let source = daemon.translations.lock().values().next().unwrap().clone(); + source.barrier().await.unwrap(); + let storage = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&env.source, &env.session_id) + )); + let partition = crate::pipeline::journal::SpanJournal::open(&storage) + .await + .unwrap(); + let through = partition.translated_through().await.unwrap(); + assert!(through > old_end); + assert_eq!(through, journal::JournalReader::recorded_len(&path).await); + drain_all(&daemon).await; + } + + #[tokio::test] + async fn startup_overflow_replays_oldest_events_before_the_buffered_tail() { + let tmp = tempfile::tempdir().unwrap(); + let (ready, startup) = oneshot::channel(); + let daemon = Daemon::new_with_startup( + debug_serve_options("test", tmp.path()), + tmp.path().into(), + Some(startup), + ); + let env = event("overflow", SessionRoute::default()); + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + let mut evicted_end = 0; + for i in 0..(INGRESS_QUEUE_CAPACITY + 6) { + let mut next = env.clone(); + next.ts_ms = i as i64; + next.payload = serde_json::json!({"index": i}); + assert!(daemon.capture_event(next).await.unwrap()); + if i == 5 { + evicted_end = journal::JournalReader::recorded_len(&path).await; + } + } + { + let state = daemon.startup_buffer.lock(); + let buffer = state.as_ref().unwrap(); + assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); + assert_eq!( + buffer.evicted.get(&path).map(|range| range.through), + Some(evicted_end) + ); + assert_eq!(buffer.events.front().unwrap().env.payload["index"], 6); + } + drain_startup_buffer(&daemon).await.unwrap(); + ready.send(()).unwrap(); + daemon.ingress_barrier().await; + let source = daemon.translations.lock().values().next().unwrap().clone(); + source.barrier().await.unwrap(); + let storage = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&env.source, &env.session_id) + )); + let db = rusqlite::Connection::open(storage).unwrap(); + let count: u64 = db + .query_row( + "SELECT count(*) FROM batches WHERE committed=1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, (INGRESS_QUEUE_CAPACITY + 6) as u64); + let partition = crate::pipeline::journal::SpanJournal::open( + &tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&env.source, &env.session_id) + )), + ) + .await + .unwrap(); + assert_eq!( + partition.translated_through().await.unwrap(), + journal::JournalReader::recorded_len(&path).await + ); + drop(db); + drain_all(&daemon).await; + } + + #[tokio::test] + async fn startup_overflow_advances_again_while_evicted_records_are_catching_up() { + let tmp = tempfile::tempdir().unwrap(); + let (ready, startup) = oneshot::channel(); + let daemon = Daemon::new_with_startup( + debug_serve_options("test", tmp.path()), + tmp.path().into(), + Some(startup), + ); + let env = event("chasing", SessionRoute::default()); + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + for i in 0..(INGRESS_QUEUE_CAPACITY + 2) { + let mut next = env.clone(); + next.ts_ms = i as i64; + next.payload = serde_json::json!({"index": i}); + assert!(daemon.capture_event(next).await.unwrap()); + } + let StartupWork::Evicted(first) = daemon.next_startup_work() else { + panic!("the oldest events were not evicted"); + }; + let first_end = first[&path].through; + for i in INGRESS_QUEUE_CAPACITY + 2..INGRESS_QUEUE_CAPACITY + 7 { + let mut next = env.clone(); + next.ts_ms = i as i64; + next.payload = serde_json::json!({"index": i}); + assert!(daemon.capture_event(next).await.unwrap()); + } + { + let state = daemon.startup_buffer.lock(); + let buffer = state.as_ref().unwrap(); + assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); + assert!(buffer.evicted[&path].through > first_end); + assert_eq!(buffer.events.front().unwrap().env.payload["index"], 7); + } + let mut cursors = HashMap::new(); + replay_evicted_startup(&daemon, &mut cursors, &first) + .await + .unwrap(); + let StartupWork::Evicted(second) = daemon.next_startup_work() else { + panic!("new evictions did not advance recovery"); + }; + replay_evicted_startup(&daemon, &mut cursors, &second) + .await + .unwrap(); + drain_startup_buffer(&daemon).await.unwrap(); + ready.send(()).unwrap(); + daemon.ingress_barrier().await; + let source = daemon.translations.lock().values().next().unwrap().clone(); + source.barrier().await.unwrap(); + let storage = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&env.source, &env.session_id) + )); + let db = rusqlite::Connection::open(storage).unwrap(); + let count: u64 = db + .query_row( + "SELECT count(*) FROM batches WHERE committed=1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, (INGRESS_QUEUE_CAPACITY + 7) as u64); + drop(db); + drain_all(&daemon).await; + } + + #[tokio::test] + async fn startup_overflow_tracks_each_source_journal_independently() { + let tmp = tempfile::tempdir().unwrap(); + let (ready, startup) = oneshot::channel(); + let daemon = Daemon::new_with_startup( + debug_serve_options("test", tmp.path()), + tmp.path().into(), + Some(startup), + ); + for i in 0..(INGRESS_QUEUE_CAPACITY + 6) { + let session = if i % 2 == 0 { "first" } else { "second" }; + let mut env = event(session, SessionRoute::default()); + env.ts_ms = i as i64; + env.payload = serde_json::json!({"index": i}); + assert!(daemon.capture_event(env).await.unwrap()); + } + { + let state = daemon.startup_buffer.lock(); + let buffer = state.as_ref().unwrap(); + assert_eq!(buffer.evicted.len(), 2); + assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); + } + drain_startup_buffer(&daemon).await.unwrap(); + ready.send(()).unwrap(); + daemon.ingress_barrier().await; + let sources: Vec<_> = daemon.translations.lock().values().cloned().collect(); + for source in sources { + source.barrier().await.unwrap(); + } + for session in ["first", "second"] { + let storage = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id("debug", session) + )); + let db = rusqlite::Connection::open(storage).unwrap(); + let count: u64 = db + .query_row( + "SELECT count(*) FROM batches WHERE committed=1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, ((INGRESS_QUEUE_CAPACITY + 6) / 2) as u64); + } + drain_all(&daemon).await; + } + + #[tokio::test] + async fn startup_overflow_keeps_pi_delta_context_across_recovery_advances() { + let tmp = tempfile::tempdir().unwrap(); + let (ready, startup) = oneshot::channel(); + let daemon = Daemon::new_with_startup( + debug_serve_options("test", tmp.path()), + tmp.path().into(), + Some(startup), + ); + let mut env = event("pi-overflow", SessionRoute::default()); + env.source = "pi".into(); + env.event = "context".into(); + for i in 0..(INGRESS_QUEUE_CAPACITY + 2) { + env.ts_ms = i as i64; + env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); + assert!(daemon.capture_event(env.clone()).await.unwrap()); + } + let StartupWork::Evicted(first) = daemon.next_startup_work() else { + panic!("missing first overflow") + }; + let mut cursors = HashMap::new(); + replay_evicted_startup(&daemon, &mut cursors, &first) + .await + .unwrap(); + for i in INGRESS_QUEUE_CAPACITY + 2..INGRESS_QUEUE_CAPACITY + 4 { + env.ts_ms = i as i64; + env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); + assert!(daemon.capture_event(env.clone()).await.unwrap()); + } + let StartupWork::Evicted(second) = daemon.next_startup_work() else { + panic!("missing second overflow") + }; + replay_evicted_startup(&daemon, &mut cursors, &second) + .await + .unwrap(); + drain_startup_buffer(&daemon).await.unwrap(); + ready.send(()).unwrap(); + daemon.ingress_barrier().await; + let source = daemon.translations.lock().values().next().unwrap().clone(); + source.barrier().await.unwrap(); + let storage = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&env.source, &env.session_id) + )); + let db = rusqlite::Connection::open(storage).unwrap(); + let count: u64 = db + .query_row( + "SELECT count(*) FROM batches WHERE committed=1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, (INGRESS_QUEUE_CAPACITY + 4) as u64); + drop(db); + drain_all(&daemon).await; + } #[tokio::test] async fn retention_requires_complete_inactive_partitions_and_nonempty_heads() { for scenario in [ diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index fba1ef20..6320a630 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -131,6 +131,10 @@ pub struct Daemon { ingress_tx: mpsc::Sender, ingress_overflow: AtomicBool, ingress_dispatched: Mutex>, + /// Earliest live append during startup, keyed by source journal path. + /// Recovery must not absorb queued live events into its historical head. + startup_cutoffs: Mutex>>, + startup_buffer: Mutex>, started: Instant, last_activity: Mutex, /// Prevents a shutdown drain from racing capture that has passed the @@ -143,8 +147,19 @@ pub struct Daemon { } impl Daemon { + #[cfg(test)] fn new(opts: ServeOptions, data_dir: PathBuf) -> Arc { + Self::new_with_startup(opts, data_dir, None) + } + + fn new_with_startup( + opts: ServeOptions, + data_dir: PathBuf, + startup: Option>, + ) -> Arc { let (ingress_tx, ingress_rx) = mpsc::channel(INGRESS_QUEUE_CAPACITY); + let startup_cutoffs = startup.as_ref().map(|_| HashMap::new()); + let startup_buffer = startup.as_ref().map(|_| StartupBuffer::new()); let daemon = Arc::new(Daemon { version: opts.version, data_dir, @@ -171,6 +186,8 @@ impl Daemon { ingress_tx, ingress_overflow: AtomicBool::new(false), ingress_dispatched: Mutex::new(HashMap::new()), + startup_cutoffs: Mutex::new(startup_cutoffs), + startup_buffer: Mutex::new(startup_buffer), started: Instant::now(), last_activity: Mutex::new(Instant::now()), capture_gate: tokio::sync::RwLock::new(()), @@ -179,7 +196,7 @@ impl Daemon { shutting_down: AtomicBool::new(false), shutdown: Notify::new(), }); - spawn_ingress_worker(daemon.clone(), ingress_rx); + spawn_ingress_worker(daemon.clone(), ingress_rx, startup); daemon } @@ -228,6 +245,12 @@ pub async fn run(args: ServeArgs, opts: ServeOptions) -> anyhow::Result<()> { paths::ensure_private_dir(parent)?; } + #[cfg(unix)] + let Some(ownership) = transport::wait_for_endpoint_lock(&socket).await? + else { + tracing::info!("another daemon owns {}; exiting", socket.display()); + return Ok(()); + }; let listener = match transport::claim(&socket, || probe_alive(&socket)).await? { Some(l) => l, None => { @@ -240,25 +263,50 @@ pub async fn run(args: ServeArgs, opts: ServeOptions) -> anyhow::Result<()> { }; tracing::info!(socket = %socket.display(), "bt-daemon listening"); - let daemon = Daemon::new(opts, data_dir); - restore_active_parent_snapshots(&daemon.data_dir, &daemon.correlation).await; - restore_pending_sessions(&daemon).await; - collect_garbage(&daemon).await; - recover_unprocessed_journals(&daemon).await; - spawn_pending_reconciler(daemon.clone()); + let (ready, startup) = oneshot::channel(); + let daemon = Daemon::new_with_startup(opts, data_dir, Some(startup)); let idle_timeout = Duration::from_secs(args.idle_timeout_secs); - spawn_idle_watchdog(daemon.clone(), idle_timeout); - spawn_session_reaper( - daemon.clone(), - Duration::from_secs(args.session_idle_timeout_secs), - ); - spawn_gc(daemon.clone()); + let session_idle_timeout = Duration::from_secs(args.session_idle_timeout_secs); + let recovery_daemon = daemon.clone(); + let recovery = tokio::spawn(async move { + restore_active_parent_snapshots(&recovery_daemon.data_dir, &recovery_daemon.correlation) + .await; + restore_pending_sessions(&recovery_daemon).await; + collect_garbage(&recovery_daemon).await; + let candidates = discover_unprocessed_journals(&recovery_daemon).await; + *recovery_daemon.startup_cutoffs.lock() = None; + for event in candidates { + dispatch_ingress_event(&recovery_daemon, event).await; + } + recovery_daemon.settle_session_actors().await; + let _ = retry_pending_sessions(&recovery_daemon).await; + if let Err(error) = drain_startup_buffer(&recovery_daemon).await { + tracing::error!(%error, "startup ingress recovery failed"); + recovery_daemon.trigger_shutdown(); + return; + } + let _ = ready.send(()); + if !recovery_daemon.shutting_down.load(Ordering::SeqCst) { + spawn_pending_reconciler(recovery_daemon.clone()); + spawn_idle_watchdog(recovery_daemon.clone(), idle_timeout); + spawn_session_reaper(recovery_daemon.clone(), session_idle_timeout); + spawn_gc(recovery_daemon); + } + }); let accept_result = accept_loop(daemon.clone(), listener).await; // Graceful drain regardless of why we stopped. + daemon.trigger_shutdown(); + if let Err(error) = recovery.await { + tracing::error!(%error, "startup recovery task failed"); + } drain_all(&daemon).await; transport::cleanup(&socket); + #[cfg(unix)] + if let Err(error) = fs2::FileExt::unlock(&ownership) { + tracing::warn!(%error, "daemon endpoint lock release failed"); + } accept_result } diff --git a/bt-daemon/src/server/recovery.rs b/bt-daemon/src/server/recovery.rs index 3a453bac..11c3b0e5 100644 --- a/bt-daemon/src/server/recovery.rs +++ b/bt-daemon/src/server/recovery.rs @@ -2,7 +2,7 @@ use super::*; -pub(super) async fn recover_unprocessed_journals(daemon: &Arc) { +pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec { let pending: Vec = daemon .pending_sessions .lock() @@ -10,7 +10,7 @@ pub(super) async fn recover_unprocessed_journals(daemon: &Arc) { .flat_map(|state| state.events.iter().cloned()) .collect(); let Ok(mut entries) = tokio::fs::read_dir(journal::journal_dir(&daemon.data_dir)).await else { - return; + return Vec::new(); }; let mut candidates = Vec::new(); while let Ok(Some(entry)) = entries.next_entry().await { @@ -19,6 +19,12 @@ pub(super) async fn recover_unprocessed_journals(daemon: &Arc) { continue; } let recorded_len = journal::JournalReader::recorded_len(&path).await; + let recorded_len = daemon + .startup_cutoffs + .lock() + .as_ref() + .and_then(|cutoffs| cutoffs.get(&path).copied()) + .map_or(recorded_len, |cutoff| recorded_len.min(cutoff)); let Ok(Some(mut reader)) = journal::JournalReader::open(&path, recorded_len).await else { continue; }; @@ -191,23 +197,25 @@ pub(super) async fn recover_unprocessed_journals(daemon: &Arc) { } candidates.sort_by_key(|event| event.env.ts_ms); let recovered = candidates.len(); - for event in candidates { + if recovered > 0 { + tracing::info!( + recovered, + "queued unprocessed journal sessions for recovery" + ); + } + candidates +} + +#[cfg(test)] +pub(super) async fn recover_unprocessed_journals(daemon: &Arc) { + for event in discover_unprocessed_journals(daemon).await { let _ = daemon .ingress_tx .send(IngressMsg::Event(Box::new(event))) .await; } - // Reconciliation stays in the daemon worker and follows every recovered - // event in queue order. Dropping the receiver is intentional: startup and - // hook capture do not wait for translation or reporting. let (reply, _ignored) = oneshot::channel(); let _ = daemon.ingress_tx.send(IngressMsg::Barrier(reply)).await; - if recovered > 0 { - tracing::info!( - recovered, - "queued unprocessed journal sessions for recovery" - ); - } } #[cfg(test)] diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index b1028f17..158c8a97 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -277,6 +277,10 @@ impl Daemon { .await?; let mut writer = writer.lock().await; let before = writer.position(); + if let Some(cutoffs) = self.startup_cutoffs.lock().as_mut() { + let path = journal::source_journal_path(&self.data_dir, &env.source, &env.session_id); + cutoffs.entry(path).or_insert(before); + } let through = writer.append(env).await?; Ok((before, through)) } diff --git a/bt-daemon/src/transport.rs b/bt-daemon/src/transport.rs index 6b7f829e..99436bb0 100644 --- a/bt-daemon/src/transport.rs +++ b/bt-daemon/src/transport.rs @@ -9,6 +9,40 @@ use std::path::Path; #[cfg(windows)] use std::time::Duration; +/// Hold a stable lock file for the lifetime of a Unix listener. A timed-out +/// RPC cannot establish that the process which bound the socket has died. +#[cfg(unix)] +pub(crate) fn lock_endpoint(endpoint: &Path) -> anyhow::Result> { + let mut name = endpoint.as_os_str().to_os_string(); + name.push(".lock"); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(std::path::PathBuf::from(name))?; + match fs2::FileExt::try_lock_exclusive(&file) { + Ok(()) => Ok(Some(file)), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => Ok(None), + Err(error) => Err(error.into()), + } +} + +/// A departing process (or a short-lived inherited file descriptor) can hold +/// the lock briefly after removing the socket. Allow that handover to finish. +#[cfg(unix)] +pub(crate) async fn wait_for_endpoint_lock( + endpoint: &Path, +) -> anyhow::Result> { + for _ in 0..50 { + if let Some(lock) = lock_endpoint(endpoint)? { + return Ok(Some(lock)); + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + Ok(None) +} + #[cfg(unix)] pub(crate) type ClientStream = tokio::net::UnixStream; #[cfg(windows)] @@ -144,14 +178,14 @@ where #[cfg(unix)] async fn claim_unix( endpoint: &Path, - probe_alive: &mut F, + _probe_alive: &mut F, ) -> anyhow::Result> where F: FnMut() -> Fut, Fut: Future, { if endpoint.exists() { - if probe_alive().await { + if unix_endpoint_in_use(endpoint).await? { return Ok(None); } cleanup(endpoint); @@ -159,7 +193,7 @@ where match Listener::bind_raw(endpoint) { Ok(listener) => Ok(Some(listener)), Err(error) if error.kind() == std::io::ErrorKind::AddrInUse => { - if probe_alive().await { + if unix_endpoint_in_use(endpoint).await? { Ok(None) } else { cleanup(endpoint); @@ -170,6 +204,25 @@ where } } +/// A successful transport connection proves ownership even when an older +/// daemon has not started accepting RPCs. A timeout is ambiguous and must not +/// authorize unlinking a potentially live listener. +#[cfg(unix)] +async fn unix_endpoint_in_use(endpoint: &Path) -> anyhow::Result { + match tokio::time::timeout(std::time::Duration::from_secs(1), connect(endpoint)).await { + Ok(Ok(_)) | Err(_) => Ok(true), + Ok(Err(error)) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused + ) => + { + Ok(false) + } + Ok(Err(error)) => Err(error.into()), + } +} + #[cfg(windows)] async fn claim_windows( endpoint: &Path, @@ -214,6 +267,56 @@ pub(crate) fn cleanup(endpoint: &Path) { #[cfg(windows)] pub(crate) fn cleanup(_endpoint: &Path) {} +#[cfg(all(test, unix))] +mod unix_tests { + #[test] + fn endpoint_lock_survives_an_unresponsive_owner_and_is_reusable() { + let tmp = tempfile::tempdir().unwrap(); + let endpoint = tmp.path().join("daemon.sock"); + let owner = super::lock_endpoint(&endpoint).unwrap().unwrap(); + assert!(super::lock_endpoint(&endpoint).unwrap().is_none()); + drop(owner); + assert!(super::lock_endpoint(&endpoint).unwrap().is_some()); + } + + #[tokio::test] + async fn startup_owner_excludes_a_contender_before_socket_bind() { + let tmp = tempfile::tempdir().unwrap(); + let endpoint = tmp.path().join("daemon.sock"); + let owner = super::lock_endpoint(&endpoint).unwrap().unwrap(); + assert!(!endpoint.exists()); + let contender_path = endpoint.clone(); + let mut contender = tokio::spawn(async move { + super::wait_for_endpoint_lock(&contender_path) + .await + .unwrap() + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), &mut contender) + .await + .is_err() + ); + assert!(!endpoint.exists()); + drop(owner); + assert!(contender.await.unwrap().is_some()); + } + + #[tokio::test] + async fn unresponsive_legacy_listener_is_not_unlinked() { + let tmp = tempfile::tempdir().unwrap(); + let endpoint = tmp.path().join("daemon.sock"); + let listener = tokio::net::UnixListener::bind(&endpoint).unwrap(); + // A legacy daemon doesn't take the new lock or answer initialize while + // recovering. Its successful transport connection still owns the path. + let claimed = super::claim(&endpoint, || async { false }).await.unwrap(); + assert!(claimed.is_none()); + assert!(endpoint.exists()); + drop(listener); + let claimed = super::claim(&endpoint, || async { false }).await.unwrap(); + assert!(claimed.is_some()); + } +} + #[cfg(all(test, windows))] mod tests { use super::*; From bd19e69448026dd5aa2a23c2d09ea6c3fc6a8fdd Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 17:52:38 +0800 Subject: [PATCH 02/19] Drain queued captures before observing transcripts --- bt-daemon/src/server/maintenance.rs | 41 +++++++++++++++++++++++++++++ bt-daemon/src/server/sessions.rs | 1 + 2 files changed, 42 insertions(+) diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 16107ef2..234ecc12 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -329,6 +329,9 @@ pub(super) async fn drain_all(daemon: &Arc) { } daemon.begin_quiesce(); let _capture_guard = daemon.capture_gate.write().await; + // Captures already acknowledged may still be waiting to register their + // transcript paths in ingress. Observe only after those have dispatched. + daemon.settle_ingress().await; daemon.observe_claude_transcripts(None, true).await; daemon.settle_ingress().await; if let Err(error) = resolve_pending_sessions(daemon, |_| true).await { @@ -469,6 +472,44 @@ mod tests { drain_all(&daemon).await; } + #[tokio::test] + async fn shutdown_observes_transcript_registered_by_queued_capture() { + let tmp = tempfile::tempdir().unwrap(); + let transcript = tmp.path().join("transcript.jsonl"); + tokio::fs::write(&transcript, b"{}\n").await.unwrap(); + let (ready, startup) = oneshot::channel(); + let daemon = Daemon::new_with_startup( + debug_serve_options("test", tmp.path()), + tmp.path().into(), + Some(startup), + ); + let mut env = event("late-transcript", SessionRoute::default()); + env.source = "claude-code".into(); + env.event = "Stop".into(); + env.payload = serde_json::json!({"transcript_path": transcript}); + daemon.capture_event(env.clone()).await.unwrap(); + tokio::fs::write(&transcript, b"{}\n{}\n").await.unwrap(); + + let shutdown_daemon = daemon.clone(); + let shutdown = tokio::spawn(async move { drain_all(&shutdown_daemon).await }); + while !daemon.quiescing.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + tokio::task::yield_now().await; + drain_startup_buffer(&daemon).await.unwrap(); + ready.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), shutdown) + .await + .unwrap() + .unwrap(); + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + let content = tokio::fs::read_to_string(path).await.unwrap(); + assert!( + content.contains("TranscriptUpdate"), + "late transcript must be journaled before shutdown" + ); + } + #[tokio::test] async fn startup_recovery_dispatches_old_session_before_new_capture() { let tmp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index 158c8a97..1d56b0bc 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -182,6 +182,7 @@ impl Daemon { /// journal file for the rest of the daemon's life; deterministic span ids /// mean a late event simply rebuilds it from the journal. pub(super) async fn retire_session(&self, key: &DeliveryKey) { + self.ingress_barrier().await; self.observe_claude_transcripts(Some(key), false).await; self.ingress_barrier().await; let lock = self.dispatch_lock(&key.source, &key.session_id); From e4413f97b5d071f911569e342fb837fbc6d086bd Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 18:36:01 +0800 Subject: [PATCH 03/19] Preserve capture order during startup overflow --- bt-daemon/src/server/ingress.rs | 161 ++++++++++++++++++++++------ bt-daemon/src/server/maintenance.rs | 115 +++++++++++++++++--- bt-daemon/tests/pipeline.rs | 5 +- 3 files changed, 230 insertions(+), 51 deletions(-) diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 47615a32..a37e1193 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -4,46 +4,116 @@ use super::*; use std::collections::VecDeque; +use std::io::{BufRead, BufReader as StdBufReader, Seek, SeekFrom, Write}; -/// Live startup captures remain ordered and bounded. Evicted entries are -/// already durable; their source journal's high-water mark is replayed first. +/// Evicted captures are indexed on disk in capture order. The index is +/// temporary: the source journals remain the durable recovery record. pub(super) struct StartupBuffer { pub(super) events: VecDeque, - pub(super) evicted: HashMap, + evicted: Option, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub(super) struct EvictedRange { pub(super) from: u64, pub(super) through: u64, } +#[derive(Serialize, Deserialize)] +struct IndexedRange { + filename: String, + range: EvictedRange, +} + +struct StartupIndex { + file: tempfile::NamedTempFile, + reader: Option, + written: u64, + read: u64, +} + +impl StartupIndex { + fn new() -> anyhow::Result { + let file = tempfile::NamedTempFile::new()?; + let reader = Some(file.reopen()?); + Ok(Self { + file, + reader, + written: 0, + read: 0, + }) + } + + fn push(&mut self, range: &IndexedRange) -> anyhow::Result<()> { + let mut line = serde_json::to_vec(range)?; + line.push(b'\n'); + self.file.write_all(&line)?; + self.written += 1; + Ok(()) + } + + fn reserve_read(&mut self) -> anyhow::Result> { + if self.read == self.written { + return Ok(None); + } + let reader = self + .reader + .take() + .ok_or_else(|| anyhow::anyhow!("startup index reader missing"))?; + self.read += 1; + Ok(Some(reader)) + } +} + +fn read_startup_index(file: std::fs::File) -> anyhow::Result<(std::fs::File, IndexedRange)> { + let mut reader = StdBufReader::new(file); + let mut line = Vec::new(); + reader.read_until(b'\n', &mut line)?; + anyhow::ensure!(line.last() == Some(&b'\n'), "truncated startup index"); + let position = reader.stream_position()?; + let mut file = reader.into_inner(); + file.seek(SeekFrom::Start(position))?; + Ok((file, serde_json::from_slice(&line)?)) +} + impl StartupBuffer { pub(super) fn new() -> Self { Self { events: VecDeque::new(), - evicted: HashMap::new(), + evicted: None, } } - fn push(&mut self, data_dir: &std::path::Path, event: PendingEvent) { + fn push(&mut self, data_dir: &std::path::Path, event: PendingEvent) -> anyhow::Result<()> { if self.events.len() == INGRESS_QUEUE_CAPACITY { - if let Some(oldest) = self.events.pop_front() { - let path = journal::source_journal_path( - data_dir, - &oldest.env.source, - &oldest.env.session_id, - ); - self.evicted - .entry(path) - .and_modify(|range| range.through = range.through.max(oldest.journal_through)) - .or_insert(EvictedRange { - from: oldest.replay_through, - through: oldest.journal_through, - }); + let oldest = self + .events + .front() + .ok_or_else(|| anyhow::anyhow!("startup buffer is empty"))?; + let path = + journal::source_journal_path(data_dir, &oldest.env.source, &oldest.env.session_id); + let range = IndexedRange { + filename: path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow::anyhow!("invalid startup journal name"))? + .to_owned(), + range: EvictedRange { + from: oldest.replay_through, + through: oldest.journal_through, + }, + }; + if self.evicted.is_none() { + self.evicted = Some(StartupIndex::new()?); } + self.evicted + .as_mut() + .ok_or_else(|| anyhow::anyhow!("startup index missing"))? + .push(&range)?; + self.events.pop_front(); } self.events.push_back(event); + Ok(()) } } @@ -54,26 +124,48 @@ pub(super) enum StartupWork { } impl Daemon { - pub(super) fn next_startup_work(&self) -> StartupWork { - let mut state = self.startup_buffer.lock(); - let Some(buffer) = state.as_mut() else { - return StartupWork::Finished; + pub(super) async fn next_startup_work(&self) -> anyhow::Result { + let reader = { + let mut state = self.startup_buffer.lock(); + let Some(buffer) = state.as_mut() else { + return Ok(StartupWork::Finished); + }; + if let Some(reader) = buffer + .evicted + .as_mut() + .map(StartupIndex::reserve_read) + .transpose()? + .flatten() + { + reader + } else if let Some(event) = buffer.events.pop_front() { + return Ok(StartupWork::Event(Box::new(event))); + } else { + *state = None; + return Ok(StartupWork::Finished); + } }; - if !buffer.evicted.is_empty() { - return StartupWork::Evicted(std::mem::take(&mut buffer.evicted)); - } - if let Some(event) = buffer.events.pop_front() { - return StartupWork::Event(Box::new(event)); - } - *state = None; - StartupWork::Finished + // Recovery reads outside the capture lock; a slow disk read cannot + // prevent hooks from enqueueing their already-journaled events. + let (reader, entry) = + tokio::task::spawn_blocking(move || read_startup_index(reader)).await??; + let mut state = self.startup_buffer.lock(); + let index = state + .as_mut() + .and_then(|buffer| buffer.evicted.as_mut()) + .ok_or_else(|| anyhow::anyhow!("startup index disappeared"))?; + index.reader = Some(reader); + Ok(StartupWork::Evicted(HashMap::from([( + journal::journal_dir(&self.data_dir).join(entry.filename), + entry.range, + )]))) } } pub(super) async fn drain_startup_buffer(daemon: &Arc) -> anyhow::Result<()> { let mut cursors = HashMap::new(); loop { - match daemon.next_startup_work() { + match daemon.next_startup_work().await? { StartupWork::Evicted(bounds) => { replay_evicted_startup(daemon, &mut cursors, &bounds).await? } @@ -510,7 +602,10 @@ impl Daemon { let event = { let mut state = self.startup_buffer.lock(); if let Some(buffer) = state.as_mut() { - buffer.push(&self.data_dir, event); + if let Err(error) = buffer.push(&self.data_dir, event) { + self.trigger_shutdown(); + return Err(format!("startup index failed: {error}")); + } return Ok(true); } event diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 234ecc12..74057603 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -425,6 +425,38 @@ mod tests { wire::Envelope, }; + #[allow(dead_code)] + mod distributed_fixture { + use crate as bt_daemon; + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/distributed.rs" + )); + } + + struct TestAuth; + + #[async_trait] + impl AuthProvider for TestAuth { + async fn resolve( + &self, + selection: &AuthSelection, + _: AuthResolveReason, + ) -> anyhow::Result { + Ok(AuthLease { + selection: selection.clone().canonicalized()?, + auth: BackendAuth { + token: "test".into(), + api_url: Some("http://127.0.0.1.invalid".into()), + app_url: None, + org_name: Some("test".into()), + org_id: Some("test".into()), + }, + expires_at_ms: None, + }) + } + } + fn event(session: &str, route: SessionRoute) -> Envelope { Envelope { source: "debug".into(), @@ -565,24 +597,16 @@ mod tests { ); let env = event("overflow", SessionRoute::default()); let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); - let mut evicted_end = 0; for i in 0..(INGRESS_QUEUE_CAPACITY + 6) { let mut next = env.clone(); next.ts_ms = i as i64; next.payload = serde_json::json!({"index": i}); assert!(daemon.capture_event(next).await.unwrap()); - if i == 5 { - evicted_end = journal::JournalReader::recorded_len(&path).await; - } } { let state = daemon.startup_buffer.lock(); let buffer = state.as_ref().unwrap(); assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); - assert_eq!( - buffer.evicted.get(&path).map(|range| range.through), - Some(evicted_end) - ); assert_eq!(buffer.events.front().unwrap().env.payload["index"], 6); } drain_startup_buffer(&daemon).await.unwrap(); @@ -629,17 +653,15 @@ mod tests { Some(startup), ); let env = event("chasing", SessionRoute::default()); - let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); for i in 0..(INGRESS_QUEUE_CAPACITY + 2) { let mut next = env.clone(); next.ts_ms = i as i64; next.payload = serde_json::json!({"index": i}); assert!(daemon.capture_event(next).await.unwrap()); } - let StartupWork::Evicted(first) = daemon.next_startup_work() else { + let StartupWork::Evicted(first) = daemon.next_startup_work().await.unwrap() else { panic!("the oldest events were not evicted"); }; - let first_end = first[&path].through; for i in INGRESS_QUEUE_CAPACITY + 2..INGRESS_QUEUE_CAPACITY + 7 { let mut next = env.clone(); next.ts_ms = i as i64; @@ -650,14 +672,13 @@ mod tests { let state = daemon.startup_buffer.lock(); let buffer = state.as_ref().unwrap(); assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); - assert!(buffer.evicted[&path].through > first_end); assert_eq!(buffer.events.front().unwrap().env.payload["index"], 7); } let mut cursors = HashMap::new(); replay_evicted_startup(&daemon, &mut cursors, &first) .await .unwrap(); - let StartupWork::Evicted(second) = daemon.next_startup_work() else { + let StartupWork::Evicted(second) = daemon.next_startup_work().await.unwrap() else { panic!("new evictions did not advance recovery"); }; replay_evicted_startup(&daemon, &mut cursors, &second) @@ -685,6 +706,69 @@ mod tests { drain_all(&daemon).await; } + #[tokio::test] + async fn startup_overflow_preserves_parent_child_tool_order() { + use distributed_fixture::{AgentKind, DistributedFixtures, ProcessTree}; + let tmp = tempfile::tempdir().unwrap(); + let (ready, startup) = oneshot::channel(); + let mut opts = debug_serve_options("test", tmp.path()); + opts.auth_provider = Some(Arc::new(TestAuth)); + let daemon = Daemon::new_with_startup(opts, tmp.path().into(), Some(startup)); + let mut fixtures = DistributedFixtures::new(); + let parent = ProcessTree::root(1000); + let child = ProcessTree::child(1002, 1001, &parent); + let mut events = + fixtures.start_turn(AgentKind::OpenCode, "parent", &parent, "delegate", 100); + events.push(fixtures.open_tool( + AgentKind::OpenCode, + "parent", + &parent, + "tool", + "do work", + 102, + )); + let child_events = + fixtures.start_turn(AgentKind::OpenCode, "child", &child, "do work", 103); + let child_key = automatic_link_key(&child_events[0]); + events.extend(child_events); + events.push(fixtures.close_tool( + AgentKind::OpenCode, + "parent", + &parent, + "tool", + "do work", + "done", + 105, + )); + for env in events { + daemon.capture_event(env).await.unwrap(); + } + for i in 0..INGRESS_QUEUE_CAPACITY { + let mut filler = event("filler", SessionRoute::default()); + filler.ts_ms = i as i64; + daemon.capture_event(filler).await.unwrap(); + } + drain_startup_buffer(&daemon).await.unwrap(); + ready.send(()).unwrap(); + daemon.settle_ingress().await; + let route = daemon + .automatic_links + .lock() + .get(&child_key) + .cloned() + .expect("child must be linked"); + let Some(crate::wire::TraceDestination::ParentSpan { components }) = route.destination + else { + panic!("child was not attached to the parent tool"); + }; + let tool_id = crate::ids::span_id( + &crate::ids::session_namespace("opencode", "parent"), + "tool:parent:tool", + ); + assert_eq!(components.span_id.as_deref(), Some(tool_id.as_str())); + drain_all(&daemon).await; + } + #[tokio::test] async fn startup_overflow_tracks_each_source_journal_independently() { let tmp = tempfile::tempdir().unwrap(); @@ -704,7 +788,6 @@ mod tests { { let state = daemon.startup_buffer.lock(); let buffer = state.as_ref().unwrap(); - assert_eq!(buffer.evicted.len(), 2); assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); } drain_startup_buffer(&daemon).await.unwrap(); @@ -749,7 +832,7 @@ mod tests { env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); assert!(daemon.capture_event(env.clone()).await.unwrap()); } - let StartupWork::Evicted(first) = daemon.next_startup_work() else { + let StartupWork::Evicted(first) = daemon.next_startup_work().await.unwrap() else { panic!("missing first overflow") }; let mut cursors = HashMap::new(); @@ -761,7 +844,7 @@ mod tests { env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); assert!(daemon.capture_event(env.clone()).await.unwrap()); } - let StartupWork::Evicted(second) = daemon.next_startup_work() else { + let StartupWork::Evicted(second) = daemon.next_startup_work().await.unwrap() else { panic!("missing second overflow") }; replay_evicted_startup(&daemon, &mut cursors, &second) diff --git a/bt-daemon/tests/pipeline.rs b/bt-daemon/tests/pipeline.rs index e6ee0146..471a49ad 100644 --- a/bt-daemon/tests/pipeline.rs +++ b/bt-daemon/tests/pipeline.rs @@ -2532,9 +2532,10 @@ async fn claude_journal_does_not_grow_with_the_transcript_on_every_turn() { // Re-journaling the transcript every turn would put this near // TURNS * transcript_len / 2 (tens of megabytes). References are ~a few - // hundred bytes each. + // hundred bytes each. Allow for longer Windows paths and concurrent + // transcript observations while remaining far below the duplicated data. assert!( - journal_len < 64 * 1024, + journal_len < 256 * 1024, "journal grew with the transcript: {journal_len} bytes for {TURNS} events \ over a {transcript_len}-byte transcript" ); From 07e59ed6fd11b6ba070064354f70aa94ee0eece9 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 19:24:26 +0800 Subject: [PATCH 04/19] Checkpoint session translation context for bounded recovery --- bt-daemon/README.md | 7 + bt-daemon/src/journal.rs | 61 ++- bt-daemon/src/muse/usage.rs | 2 +- bt-daemon/src/pipeline/journal.rs | 84 +++- bt-daemon/src/pipeline/prepare.rs | 13 + bt-daemon/src/pipeline/translation.rs | 465 +++++++++++++++++++++- bt-daemon/src/server/ingress.rs | 288 ++++++-------- bt-daemon/src/server/maintenance.rs | 3 + bt-daemon/src/server/mod.rs | 5 +- bt-daemon/src/server/recovery.rs | 149 +++++++ bt-daemon/src/server/sessions.rs | 5 +- bt-daemon/src/translate/antigravity.rs | 6 + bt-daemon/src/translate/checkpoint.rs | 67 ++++ bt-daemon/src/translate/claude.rs | 16 +- bt-daemon/src/translate/codex.rs | 13 +- bt-daemon/src/translate/cursor/mod.rs | 15 +- bt-daemon/src/translate/debug.rs | 2 + bt-daemon/src/translate/grok.rs | 13 +- bt-daemon/src/translate/mod.rs | 24 ++ bt-daemon/src/translate/muse.rs | 5 +- bt-daemon/src/translate/muse_events.rs | 2 +- bt-daemon/src/translate/muse_usage.rs | 2 +- bt-daemon/src/translate/opencode.rs | 12 +- bt-daemon/src/translate/pi.rs | 8 +- bt-daemon/src/translate/recent.rs | 4 + bt-daemon/src/translate/tool.rs | 2 +- bt-daemon/src/translate/turn_lineage.rs | 7 + bt-daemon/tests/translator_checkpoints.rs | 202 ++++++++++ 28 files changed, 1286 insertions(+), 196 deletions(-) create mode 100644 bt-daemon/src/translate/checkpoint.rs create mode 100644 bt-daemon/tests/translator_checkpoints.rs diff --git a/bt-daemon/README.md b/bt-daemon/README.md index 0a1c5127..311ec539 100644 --- a/bt-daemon/README.md +++ b/bt-daemon/README.md @@ -58,6 +58,13 @@ or `bt trace doctor ` for configuration, authentication, and recent session diagnostics. Hook exit codes can be observed by the caller while the coding agent decides whether to continue its turn. +Hooks continue recording events while the daemon recovers earlier work. Each +source session has one translation position and a periodic context checkpoint +in its existing SQLite store; each destination has its own SpanOp delivery +position. Recovery restores open turns, tools, and transcript cursors from the +checkpoint and replays only the remaining suffix. An unavailable or incompatible +checkpoint falls back to rebuilding from the event journal. + ## Per-agent plugin settings Each coding agent reads an independent non-credential `braintrust.json` file: diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index c301022b..b5751eaf 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -14,7 +14,7 @@ use crate::wire::{BackendAuth, Envelope, RedactedEnvelope, SessionRoute}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::path::{Path, PathBuf}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; pub fn journal_dir(data_dir: &Path) -> PathBuf { data_dir.join("journal") @@ -253,10 +253,29 @@ impl JournalWriter { /// bounded instead by writing each transcript byte once (see /// [`crate::transcript_mirror`]) and by age-based GC, and replay reads it /// as a stream so a large journal never becomes a large allocation. + #[cfg(test)] pub async fn append(&mut self, env: &Envelope) -> anyhow::Result { + self.append_ordered(env, None).await + } + + pub(crate) async fn append_ordered( + &mut self, + env: &Envelope, + sequence: Option, + ) -> anyhow::Result { + #[derive(Serialize)] + struct Captured<'a> { + #[serde(flatten)] + envelope: &'a RedactedEnvelope, + #[serde(skip_serializing_if = "Option::is_none")] + _bt_capture_sequence: Option, + } let mut redacted = env.redacted(); compact_pi_payload(&mut redacted, &mut self.pi_context); - let mut line = serde_json::to_vec(&redacted)?; + let mut line = serde_json::to_vec(&Captured { + envelope: &redacted, + _bt_capture_sequence: sequence, + })?; line.push(b'\n'); self.file.write_all(&line).await?; self.file.flush().await?; @@ -318,6 +337,8 @@ pub struct JournalRecordEntry { pub record: JournalRecord, /// Exclusive byte offset after this record in the journal. pub through: u64, + /// Daemon-local capture order. Kept outside the native event payload. + pub(crate) capture_sequence: Option, } /// Streaming reader over one session's journal. @@ -333,6 +354,38 @@ pub struct JournalReader { } impl JournalReader { + pub(crate) fn decoder_context(&self) -> &[serde_json::Value] { + &self.pi_context + } + + /// Resume at a committed record boundary with the Pi delta decoder's base. + pub(crate) async fn resume( + path: &Path, + position: u64, + through: u64, + pi_context: Vec, + ) -> anyhow::Result { + let mut file = tokio::fs::File::open(path).await?; + anyhow::ensure!( + position <= through && through <= file.metadata().await?.len(), + "checkpoint is beyond the source journal" + ); + if position > 0 { + file.seek(std::io::SeekFrom::Start(position - 1)).await?; + anyhow::ensure!( + file.read_u8().await? == b'\n', + "checkpoint is not at a journal record boundary" + ); + } + file.seek(std::io::SeekFrom::Start(position)).await?; + Ok(Self { + reader: tokio::io::BufReader::new(file.take(through - position)), + path: path.to_owned(), + line_no: 0, + position, + pi_context, + }) + } /// Open a journal for streaming, reading at most `through` bytes. /// /// That bound is what keeps replay from consuming the very event that @@ -396,6 +449,9 @@ impl JournalReader { let value: serde_json::Value = serde_json::from_str(line).map_err(|error| { anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) })?; + let capture_sequence = value + .get("_bt_capture_sequence") + .and_then(serde_json::Value::as_u64); let record = if value .get("_bt_record_type") .and_then(serde_json::Value::as_str) @@ -427,6 +483,7 @@ impl JournalReader { return Ok(Some(JournalRecordEntry { record, through: self.position, + capture_sequence, })); } } diff --git a/bt-daemon/src/muse/usage.rs b/bt-daemon/src/muse/usage.rs index e2af3a67..5b282690 100644 --- a/bt-daemon/src/muse/usage.rs +++ b/bt-daemon/src/muse/usage.rs @@ -26,7 +26,7 @@ pub(crate) enum UsageScope { Model, } -#[derive(Debug, Clone, Default, Serialize, PartialEq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] pub(crate) struct TokenMetrics { #[serde(skip_serializing_if = "Option::is_none")] pub prompt_tokens: Option, diff --git a/bt-daemon/src/pipeline/journal.rs b/bt-daemon/src/pipeline/journal.rs index 0bbff985..4fa91e22 100644 --- a/bt-daemon/src/pipeline/journal.rs +++ b/bt-daemon/src/pipeline/journal.rs @@ -45,6 +45,13 @@ pub(crate) struct SpanJournal { payload_reads: Arc, } +#[derive(Clone)] +pub(crate) struct TranslationCheckpoint { + pub input: u64, + pub version: String, + pub state: Vec, +} + /// Explicit codec preserves fields intentionally omitted from wire/plugin JSON. #[derive(Serialize, Deserialize)] struct StoredOp { @@ -181,7 +188,7 @@ impl SpanJournal { let db = Connection::open(path)?; db.busy_timeout(std::time::Duration::from_secs(5))?; let version: u32 = db.query_row("PRAGMA user_version", [], |row| row.get(0))?; - anyhow::ensure!(version <= 6, "unsupported SpanOp journal schema {version}"); + anyhow::ensure!(version <= 7, "unsupported SpanOp journal schema {version}"); db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;")?; if version < 3 { db.execute_batch( @@ -233,6 +240,11 @@ impl SpanJournal { prune(&db)?; db.execute_batch("PRAGMA user_version=6; COMMIT")?; } + if version < 7 { + db.execute_batch("BEGIN IMMEDIATE; + CREATE TABLE IF NOT EXISTS translation_checkpoint (id INTEGER PRIMARY KEY CHECK(id=1), input INTEGER NOT NULL, version TEXT NOT NULL, state BLOB NOT NULL); + PRAGMA user_version=7; COMMIT")?; + } let tail = db.query_row("SELECT tail FROM publication WHERE id=1", [], |row| row.get(0))?; Ok((db, tail)) }) @@ -294,6 +306,51 @@ impl SpanJournal { .await } + pub async fn translation_checkpoint(&self) -> anyhow::Result> { + self.with_db(|db| { + Ok(db + .query_row( + "SELECT input,version,state FROM translation_checkpoint WHERE id=1", + [], + |row| { + Ok(TranslationCheckpoint { + input: row.get(0)?, + version: row.get(1)?, + state: row.get(2)?, + }) + }, + ) + .optional()?) + }) + .await + } + + /// Refresh idle context only at the already committed translation head. + pub async fn save_translation_checkpoint( + &self, + checkpoint: TranslationCheckpoint, + ) -> anyhow::Result<()> { + self.with_db(move |db| { + let tx = db.transaction()?; + let head: u64 = tx.query_row( + "SELECT coalesce(max(input),0) FROM batches WHERE committed=1", + [], + |row| row.get(0), + )?; + anyhow::ensure!( + checkpoint.input == head, + "checkpoint must match the translation head" + ); + tx.execute( + "INSERT OR REPLACE INTO translation_checkpoint VALUES(1,?,?,?)", + params![checkpoint.input, checkpoint.version, checkpoint.state], + )?; + tx.commit()?; + Ok(()) + }) + .await + } + /// Restart abandons only invisible receipts; a committed identity is never rewritten. pub async fn begin(&self, input: u64, observation: bool) -> anyhow::Result { self.with_db(move |db| { @@ -359,7 +416,8 @@ impl SpanJournal { let first = *ordinal; let positions = self .with_db(move |db| { - let tx = db.transaction()?; + let tx = + db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let mut positions = Vec::new(); let (committed, observation): (bool, bool) = tx.query_row( "SELECT committed,observation FROM batches WHERE input=?", @@ -493,14 +551,24 @@ impl SpanJournal { } pub async fn commit(&self, input: u64) -> anyhow::Result { - self.commit_output(input, None).await + self.commit_output(input, None, None).await } /// Root candidates and translated output share the consumed-input boundary. + #[cfg(test)] pub async fn commit_with_roots( &self, input: u64, roots: &BTreeMap>, + ) -> anyhow::Result { + self.commit_with_context(input, roots, None).await + } + + pub async fn commit_with_context( + &self, + input: u64, + roots: &BTreeMap>, + checkpoint: Option, ) -> anyhow::Result { let roots = roots .iter() @@ -512,7 +580,7 @@ impl SpanJournal { Ok((id.clone(), serde_json::to_vec(&ops)?)) }) .collect::>>()?; - self.commit_output(input, Some(roots)).await + self.commit_output(input, Some(roots), checkpoint).await } pub async fn root_candidates(&self) -> anyhow::Result>> { @@ -542,6 +610,7 @@ impl SpanJournal { &self, input: u64, roots: Option)>>, + checkpoint: Option, ) -> anyhow::Result { let tail = self .with_db(move |db| { @@ -565,6 +634,13 @@ impl SpanJournal { tx.execute("UPDATE batches SET committed=1 WHERE input=?", [input])? == 1, "missing translation batch" ); + if let Some(checkpoint) = checkpoint { + anyhow::ensure!(checkpoint.input == input, "checkpoint input mismatch"); + tx.execute( + "INSERT OR REPLACE INTO translation_checkpoint VALUES(1,?,?,?)", + params![input, checkpoint.version, checkpoint.state], + )?; + } if let Some(roots) = roots { tx.execute("DELETE FROM root_candidates", [])?; for (id, value) in roots { diff --git a/bt-daemon/src/pipeline/prepare.rs b/bt-daemon/src/pipeline/prepare.rs index 4148bfc0..76783b53 100644 --- a/bt-daemon/src/pipeline/prepare.rs +++ b/bt-daemon/src/pipeline/prepare.rs @@ -5,6 +5,7 @@ use crate::{ }; use std::collections::HashSet; +#[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct Prepare { source: String, roots: HashSet, @@ -129,6 +130,18 @@ impl ConfiguredTranslator { } impl crate::translate::AgentTranslator for ConfiguredTranslator { + fn save_state(&self) -> anyhow::Result>> { + let Some(state) = self.inner.save_state()? else { + return Ok(None); + }; + Ok(Some(serde_json::to_vec(&(state, &self.prepare))?)) + } + fn restore_state(&mut self, state: &[u8]) -> anyhow::Result<()> { + let (inner, prepare): (Vec, Prepare) = serde_json::from_slice(state)?; + self.inner.restore_state(&inner)?; + self.prepare = prepare; + Ok(()) + } fn handle( &mut self, env: &crate::wire::Envelope, diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 779667b9..7ce12a3e 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -1,5 +1,5 @@ //! One ordered source consumer. Output publication is independent of route delivery. -use super::journal::SpanJournal; +use super::journal::{SpanJournal, TranslationCheckpoint}; use super::prepare::Prepare; use super::roots::Roots; use super::{ @@ -22,6 +22,39 @@ use std::{ }; use tokio::sync::{mpsc, oneshot, Notify}; +const CHECKPOINT_EVENTS: u64 = 64; +const CHECKPOINT_BYTES: u64 = 1024 * 1024; + +#[derive(serde::Serialize, serde::Deserialize)] +pub(crate) struct RecoveryContext { + format: u32, + translator: Box, + pub decoder: Vec, + capture: Option, + bounds: HashMap, + pub templates: HashMap, +} + +impl RecoveryContext { + pub fn decode( + checkpoint: &TranslationCheckpoint, + version: &str, + through: u64, + ) -> anyhow::Result { + anyhow::ensure!( + checkpoint.version == version && checkpoint.input <= through, + "translation checkpoint is incompatible with the current head" + ); + let decoder = zstd::stream::read::Decoder::new(checkpoint.state.as_slice())?; + let context: Self = serde_json::from_reader(std::io::BufReader::new(decoder))?; + anyhow::ensure!( + context.format == 1, + "unsupported translation context format" + ); + Ok(context) + } +} + #[derive(Clone)] pub(crate) struct Observation { pub key: String, @@ -113,8 +146,12 @@ impl Translation { capture: None, pending_snapshots: HashSet::new(), bounds: HashMap::new(), + templates: HashMap::new(), position: 0, committed, + checkpoint_position: 0, + checkpoint_events: 0, + checkpoint_safe: true, state, publication: None, phase: Boundary::Journal, @@ -248,8 +285,12 @@ struct Producer { capture: Option, pending_snapshots: HashSet, bounds: HashMap, + templates: HashMap, position: u64, committed: u64, + checkpoint_position: u64, + checkpoint_events: u64, + checkpoint_safe: bool, state: State, publication: Option, phase: Boundary, @@ -396,6 +437,76 @@ impl Producer { self.state = State::Recovering(self.failure(error, attempts)); } } + if self.publication.is_none() && self.position == self.committed { + if let Err(error) = self.persist_current_context().await { + tracing::warn!(%error, "translation checkpoint could not be saved at shutdown"); + } + } + } + + async fn checkpoint_context( + &mut self, + input: u64, + ) -> anyhow::Result> { + if !self.checkpoint_safe || self.translator.is_none() || self.reader.is_none() { + return Ok(None); + } + let translator = self + .translator + .take() + .ok_or_else(|| anyhow::anyhow!("translator is missing"))?; + let reader = self + .reader + .take() + .ok_or_else(|| anyhow::anyhow!("source reader is missing"))?; + let capture = self.capture.clone(); + let bounds = self.bounds.clone(); + let templates = self.templates.clone(); + let version = self.options.version.clone(); + // Context may include a long active conversation. Its encoding and + // compression must not monopolize a runtime thread serving hook IPC. + let (translator, reader, checkpoint) = tokio::task::spawn_blocking(move || { + let checkpoint = (|| -> anyhow::Result> { + let Some(state) = translator.save_state()? else { + return Ok(None); + }; + let state = RecoveryContext { + format: 1, + translator: serde_json::value::RawValue::from_string(String::from_utf8( + state, + )?)?, + decoder: reader.decoder_context().to_vec(), + capture, + bounds, + templates, + }; + let mut encoder = zstd::stream::Encoder::new(Vec::new(), 1)?; + serde_json::to_writer(&mut encoder, &state)?; + let state = encoder.finish()?; + Ok(Some(TranslationCheckpoint { + input, + version, + state, + })) + })(); + (translator, reader, checkpoint) + }) + .await?; + self.translator = Some(translator); + self.reader = Some(reader); + checkpoint + } + + async fn persist_current_context(&mut self) -> anyhow::Result<()> { + if self.checkpoint_position == self.committed { + return Ok(()); + } + if let Some(checkpoint) = self.checkpoint_context(self.committed).await? { + self.journal.save_translation_checkpoint(checkpoint).await?; + self.checkpoint_position = self.committed; + self.checkpoint_events = 0; + } + Ok(()) } fn failure(&self, error: anyhow::Error, attempt: u32) -> Failure { let position = self.committed.saturating_add(1); @@ -479,6 +590,54 @@ impl Producer { &self.options.namespace, )?, ); + self.position = 0; + self.checkpoint_position = 0; + self.checkpoint_events = 0; + self.checkpoint_safe = true; + if let Some(checkpoint) = self.journal.translation_checkpoint().await? { + if checkpoint.version == self.options.version && checkpoint.input <= self.committed { + let restored = async { + let registry = self.options.registry.clone(); + let source = self.options.source.clone(); + let namespace = self.options.namespace.clone(); + let version = self.options.version.clone(); + let committed = self.committed; + let input = checkpoint.input; + let (translator, context) = + tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let context = + RecoveryContext::decode(&checkpoint, &version, committed)?; + let mut translator = registry + .create_checked_with_session_namespace(&source, &namespace)?; + translator.restore_state(context.translator.get().as_bytes())?; + Ok((translator, context)) + }) + .await??; + let reader = JournalReader::resume( + &self.options.event_path, + input, + self.committed, + context.decoder, + ) + .await?; + self.translator = Some(translator); + self.reader = Some(reader); + self.capture = context.capture; + self.bounds = context.bounds; + self.templates = context.templates; + self.position = input; + self.checkpoint_position = input; + Ok::<_, anyhow::Error>(()) + } + .await; + match restored { + Ok(()) => return Ok(()), + Err(error) => { + tracing::warn!(%error, "invalid translation checkpoint; rebuilding from journal") + } + } + } + } self.reader = JournalReader::open(&self.options.event_path, self.committed).await?; Ok(()) } @@ -521,6 +680,33 @@ impl Producer { let mut env = crate::journal::envelope_from_redacted(event); if let Some(route) = &env.route { self.journal.register(route).await?; + // Discovery needs subscriptions and transcript references, + // not copies of native prompts, responses, or credentials. + let mut template = env.redacted(); + template.payload = env + .payload + .as_object() + .map(|payload| { + serde_json::Value::Object( + payload + .iter() + .filter(|(key, _)| { + matches!( + key.as_str(), + "transcript_path" + | "agent_transcript_path" + | "cwd" + | "_bt_transcript_mirror" + | "_bt_claude_transcript_mirrors" + ) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + ) + }) + .unwrap_or_else(|| serde_json::json!({})); + self.templates + .insert(serde_json::to_string(route)?, template); } if crate::translate::canonical_source_name(&env.source) != self.options.source { self.position = input; @@ -566,6 +752,12 @@ impl Producer { self.position = input; continue; } + if matches!(control.kind, ControlKind::Retire) { + // Retirement emits defensive ends without making them + // part of the logical state restored on a later resume. + self.persist_current_context().await?; + self.checkpoint_safe = false; + } self.phase = Boundary::Translation; let translator = self .translator @@ -698,9 +890,23 @@ impl Producer { } if publish { self.phase = Boundary::Journal; + let checkpoint = if self.checkpoint_events + 1 >= CHECKPOINT_EVENTS + || pending.input.saturating_sub(self.checkpoint_position) >= CHECKPOINT_BYTES + { + self.checkpoint_context(pending.input).await? + } else { + None + }; + let checkpointed = checkpoint.is_some(); self.journal - .commit_with_roots(pending.input, self.roots.pending()) + .commit_with_context(pending.input, self.roots.pending(), checkpoint) .await?; + if checkpointed { + self.checkpoint_position = pending.input; + self.checkpoint_events = 0; + } else { + self.checkpoint_events += 1; + } self.committed = pending.input; self.progress .committed @@ -774,6 +980,257 @@ mod tests { use super::*; use crate::{pipeline::delivery::InputPort, translate::SpanRow}; + fn resume_options( + data_dir: &std::path::Path, + source: &str, + events: Arc>, + ) -> TranslationOptions { + TranslationOptions { + source: source.into(), + session_id: "checkpoint".into(), + namespace: crate::ids::session_namespace(source, "checkpoint"), + version: "test".into(), + data_dir: data_dir.into(), + events, + event_path: crate::journal::source_journal_path(data_dir, source, "checkpoint"), + registry: Arc::new(Registry::default_agents()), + correlation: Arc::default(), + correlation_changed: Arc::default(), + } + } + + struct BlockedCheckpoint { + entered: Arc, + release: std::sync::mpsc::Receiver<()>, + } + impl AgentTranslator for BlockedCheckpoint { + fn handle( + &mut self, + _: &crate::wire::Envelope, + _: &SessionCtx, + ) -> anyhow::Result> { + Ok(Vec::new()) + } + fn save_state(&self) -> anyhow::Result>> { + self.entered.notify_one(); + self.release + .recv_timeout(std::time::Duration::from_secs(10))?; + Ok(Some(b"{}".to_vec())) + } + } + + #[tokio::test] + async fn slow_checkpoint_encoding_does_not_block_journal_capture_on_the_runtime() { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let entered = Arc::new(Notify::new()); + let (release, resume) = std::sync::mpsc::channel(); + let mut producer = Producer { + options: resume_options(tmp.path(), "debug", writer.clone()), + journal: SpanJournal::open(&tmp.path().join("checkpoint.sqlite")) + .await + .unwrap(), + progress: Arc::default(), + error: Arc::default(), + recovery: Arc::default(), + translator: Some(Box::new(BlockedCheckpoint { + entered: entered.clone(), + release: resume, + })), + reader: JournalReader::open(&path, 0).await.unwrap(), + observers: Vec::new(), + origins: WeakOrigins::default(), + roots: Roots::default(), + capture: None, + pending_snapshots: HashSet::new(), + bounds: HashMap::new(), + templates: HashMap::new(), + position: 0, + committed: 0, + checkpoint_position: 0, + checkpoint_events: 0, + checkpoint_safe: true, + state: State::Processing, + publication: None, + phase: Boundary::Journal, + }; + let checkpoint = tokio::spawn(async move { producer.checkpoint_context(0).await }); + entered.notified().await; + let env = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"during-checkpoint","ts_ms":1,"payload":{}})).unwrap(); + let through = tokio::time::timeout(std::time::Duration::from_secs(2), async { + writer.lock().await.append(&env).await + }) + .await + .unwrap() + .unwrap(); + assert!(through > 0); + release.send(()).unwrap(); + assert!(checkpoint.await.unwrap().unwrap().is_some()); + } + + #[tokio::test] + async fn checkpoint_seeks_past_completed_history_and_restores_pi_delta_context() { + for source in ["debug", "pi"] { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), source, "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let translation = + Translation::spawn(resume_options(tmp.path(), source, writer.clone())) + .await + .unwrap(); + let mut env = crate::wire::Envelope { + source: source.into(), + session_id: "checkpoint".into(), + event: "context".into(), + ts_ms: 0, + payload: serde_json::json!({}), + route: Some(SessionRoute::default()), + config: None, + managed_run_id: None, + capture: None, + source_version: None, + plugin_version: None, + }; + let mut boundary = 0; + for i in 0..CHECKPOINT_EVENTS + 3 { + env.ts_ms = i as i64; + env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); + boundary = writer.lock().await.append(&env).await.unwrap(); + } + translation.advance(boundary).await.unwrap(); + translation.barrier().await.unwrap(); + let checkpoint = translation + .journal + .translation_checkpoint() + .await + .unwrap() + .unwrap(); + assert!( + checkpoint.input < boundary, + "test must replay a suffix after a periodic checkpoint" + ); + let task = translation.task.lock().await.take().unwrap(); + task.abort(); + let _ = task.await; // crash, without the idle checkpoint + // Keep boundaries intact but make all pre-checkpoint JSON invalid. + // Recovery must seek over this prefix rather than decode/retranslate it. + let mut contents = tokio::fs::read(&path).await.unwrap(); + for byte in &mut contents[..checkpoint.input as usize] { + if *byte != b'\n' { + *byte = b'x'; + } + } + tokio::fs::write(&path, contents).await.unwrap(); + env.ts_ms += 1; + env.payload["event"]["messages"][1]["content"] = serde_json::json!("after restart"); + let tail = writer.lock().await.append(&env).await.unwrap(); + let resumed = Translation::spawn(resume_options(tmp.path(), source, writer.clone())) + .await + .unwrap(); + resumed.advance(tail).await.unwrap(); + resumed.barrier().await.unwrap(); + assert!( + resumed.error.lock().is_none(), + "{source}: {:?}", + resumed.error.lock() + ); + assert_eq!(resumed.journal.translated_through().await.unwrap(), tail); + resumed.shutdown().await; + } + } + + #[tokio::test] + async fn invalid_or_old_context_rebuilds_safely_from_the_source_journal() { + for alteration in [ + "UPDATE translation_checkpoint SET state=x'00'", + "UPDATE translation_checkpoint SET version='old'", + ] { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let env: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ + "source":"debug", "session_id":"checkpoint", "event":"event", "ts_ms":1,"payload":{}, + })).unwrap(); + let translation = + Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) + .await + .unwrap(); + let mut tail = 0; + for _ in 0..CHECKPOINT_EVENTS { + tail = writer.lock().await.append(&env).await.unwrap(); + } + translation.advance(tail).await.unwrap(); + translation.barrier().await.unwrap(); + translation.shutdown().await; + let storage = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id("debug", "checkpoint") + )); + let db = rusqlite::Connection::open(storage).unwrap(); + db.execute_batch(alteration).unwrap(); + tail = writer.lock().await.append(&env).await.unwrap(); + let resumed = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) + .await + .unwrap(); + resumed.advance(tail).await.unwrap(); + resumed.barrier().await.unwrap(); + assert!(resumed.error.lock().is_none()); + assert_eq!(resumed.journal.translated_through().await.unwrap(), tail); + let batches: u64 = db + .query_row( + "SELECT count(*) FROM batches WHERE committed=1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(batches, CHECKPOINT_EVENTS + 1); + resumed.shutdown().await; + } + } + + #[tokio::test] + async fn translation_checkpoint_commit_failure_does_not_move_the_source_head() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("atomic.sqlite"); + let journal = SpanJournal::open(&path).await.unwrap(); + journal.begin(10, true).await.unwrap(); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch("CREATE TRIGGER fail_context BEFORE INSERT ON translation_checkpoint BEGIN SELECT RAISE(FAIL,'checkpoint write failed'); END;").unwrap(); + let context = TranslationCheckpoint { + input: 10, + version: "test".into(), + state: vec![1, 2], + }; + assert!(journal + .commit_with_context(10, &Default::default(), Some(context.clone())) + .await + .is_err()); + assert_eq!(journal.translated_through().await.unwrap(), 0); + assert!(journal.translation_checkpoint().await.unwrap().is_none()); + db.execute_batch("DROP TRIGGER fail_context;").unwrap(); + journal + .commit_with_context(10, &Default::default(), Some(context)) + .await + .unwrap(); + assert_eq!(journal.translated_through().await.unwrap(), 10); + assert_eq!( + journal + .translation_checkpoint() + .await + .unwrap() + .unwrap() + .input, + 10 + ); + } + #[tokio::test] async fn shared_storage_stays_open_until_the_last_route_finishes_draining() { let tmp = tempfile::tempdir().unwrap(); @@ -984,8 +1441,12 @@ mod tests { capture: None, pending_snapshots: HashSet::new(), bounds: HashMap::new(), + templates: HashMap::new(), position: 0, committed: 0, + checkpoint_position: 0, + checkpoint_events: 0, + checkpoint_safe: true, state: State::Processing, phase: Boundary::Journal, publication: Some(Publication { diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index a37e1193..04d6a1a6 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -4,113 +4,43 @@ use super::*; use std::collections::VecDeque; -use std::io::{BufRead, BufReader as StdBufReader, Seek, SeekFrom, Write}; - -/// Evicted captures are indexed on disk in capture order. The index is -/// temporary: the source journals remain the durable recovery record. +/// Live startup captures remain ordered and bounded. Evicted entries are +/// already durable; their source journal's high-water mark is replayed first. pub(super) struct StartupBuffer { pub(super) events: VecDeque, - evicted: Option, + pub(super) evicted: HashMap, } -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug)] pub(super) struct EvictedRange { pub(super) from: u64, pub(super) through: u64, } -#[derive(Serialize, Deserialize)] -struct IndexedRange { - filename: String, - range: EvictedRange, -} - -struct StartupIndex { - file: tempfile::NamedTempFile, - reader: Option, - written: u64, - read: u64, -} - -impl StartupIndex { - fn new() -> anyhow::Result { - let file = tempfile::NamedTempFile::new()?; - let reader = Some(file.reopen()?); - Ok(Self { - file, - reader, - written: 0, - read: 0, - }) - } - - fn push(&mut self, range: &IndexedRange) -> anyhow::Result<()> { - let mut line = serde_json::to_vec(range)?; - line.push(b'\n'); - self.file.write_all(&line)?; - self.written += 1; - Ok(()) - } - - fn reserve_read(&mut self) -> anyhow::Result> { - if self.read == self.written { - return Ok(None); - } - let reader = self - .reader - .take() - .ok_or_else(|| anyhow::anyhow!("startup index reader missing"))?; - self.read += 1; - Ok(Some(reader)) - } -} - -fn read_startup_index(file: std::fs::File) -> anyhow::Result<(std::fs::File, IndexedRange)> { - let mut reader = StdBufReader::new(file); - let mut line = Vec::new(); - reader.read_until(b'\n', &mut line)?; - anyhow::ensure!(line.last() == Some(&b'\n'), "truncated startup index"); - let position = reader.stream_position()?; - let mut file = reader.into_inner(); - file.seek(SeekFrom::Start(position))?; - Ok((file, serde_json::from_slice(&line)?)) -} - impl StartupBuffer { pub(super) fn new() -> Self { Self { events: VecDeque::new(), - evicted: None, + evicted: HashMap::new(), } } fn push(&mut self, data_dir: &std::path::Path, event: PendingEvent) -> anyhow::Result<()> { if self.events.len() == INGRESS_QUEUE_CAPACITY { - let oldest = self - .events - .front() - .ok_or_else(|| anyhow::anyhow!("startup buffer is empty"))?; - let path = - journal::source_journal_path(data_dir, &oldest.env.source, &oldest.env.session_id); - let range = IndexedRange { - filename: path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| anyhow::anyhow!("invalid startup journal name"))? - .to_owned(), - range: EvictedRange { - from: oldest.replay_through, - through: oldest.journal_through, - }, - }; - if self.evicted.is_none() { - self.evicted = Some(StartupIndex::new()?); + if let Some(oldest) = self.events.pop_front() { + let path = journal::source_journal_path( + data_dir, + &oldest.env.source, + &oldest.env.session_id, + ); + self.evicted + .entry(path) + .and_modify(|range| range.through = range.through.max(oldest.journal_through)) + .or_insert(EvictedRange { + from: oldest.replay_through, + through: oldest.journal_through, + }); } - self.evicted - .as_mut() - .ok_or_else(|| anyhow::anyhow!("startup index missing"))? - .push(&range)?; - self.events.pop_front(); } self.events.push_back(event); Ok(()) @@ -125,40 +55,18 @@ pub(super) enum StartupWork { impl Daemon { pub(super) async fn next_startup_work(&self) -> anyhow::Result { - let reader = { - let mut state = self.startup_buffer.lock(); - let Some(buffer) = state.as_mut() else { - return Ok(StartupWork::Finished); - }; - if let Some(reader) = buffer - .evicted - .as_mut() - .map(StartupIndex::reserve_read) - .transpose()? - .flatten() - { - reader - } else if let Some(event) = buffer.events.pop_front() { - return Ok(StartupWork::Event(Box::new(event))); - } else { - *state = None; - return Ok(StartupWork::Finished); - } - }; - // Recovery reads outside the capture lock; a slow disk read cannot - // prevent hooks from enqueueing their already-journaled events. - let (reader, entry) = - tokio::task::spawn_blocking(move || read_startup_index(reader)).await??; let mut state = self.startup_buffer.lock(); - let index = state - .as_mut() - .and_then(|buffer| buffer.evicted.as_mut()) - .ok_or_else(|| anyhow::anyhow!("startup index disappeared"))?; - index.reader = Some(reader); - Ok(StartupWork::Evicted(HashMap::from([( - journal::journal_dir(&self.data_dir).join(entry.filename), - entry.range, - )]))) + let Some(buffer) = state.as_mut() else { + return Ok(StartupWork::Finished); + }; + if !buffer.evicted.is_empty() { + return Ok(StartupWork::Evicted(std::mem::take(&mut buffer.evicted))); + } + if let Some(event) = buffer.events.pop_front() { + return Ok(StartupWork::Event(Box::new(event))); + } + *state = None; + Ok(StartupWork::Finished) } } @@ -194,16 +102,16 @@ pub(super) async fn replay_evicted_startup( "evicted startup journal is truncated" ); if !cursors.contains_key(path) { - let reader = journal::JournalReader::open(path, range.from) - .await? - .ok_or_else(|| anyhow::anyhow!("evicted startup journal disappeared"))?; - cursors.insert( - path.clone(), - StartupCursor { - reader, - position: 0, - }, - ); + let (reader, position) = match resume_discovery(daemon, path, range.from).await { + Ok(Some((reader, position, _))) => (reader, position), + _ => ( + journal::JournalReader::open(path, range.from) + .await? + .ok_or_else(|| anyhow::anyhow!("evicted startup journal disappeared"))?, + 0, + ), + }; + cursors.insert(path.clone(), StartupCursor { reader, position }); } let cursor = cursors .get_mut(path) @@ -225,40 +133,77 @@ pub(super) async fn replay_evicted_startup( "invalid evicted startup boundary" ); cursor.reader.extend(range.through); - while cursor.position < range.through { - let before = cursor.position; - let entry = cursor - .reader - .next_record() - .await? - .ok_or_else(|| anyhow::anyhow!("evicted startup range ends inside a record"))?; - cursor.position = entry.through; - if let journal::JournalRecord::Event(redacted) = entry.record { - let mut env = journal::envelope_from_redacted(redacted); - let Some(source) = daemon.translators.canonical_source(&env.source) else { - continue; - }; - env.source = source.to_owned(); - dispatch_ingress_event( - daemon, - PendingEvent { - env, - replay_through: before, - journal_through: entry.through, - settings_through: None, - }, - ) - .await; - } + } + // Keep one lookahead per source. The sequence is persisted in the same + // NDJSON append as its event, so capture needs no secondary write queue. + let mut ready = Vec::new(); + for (path, range) in bounds { + let cursor = cursors + .get_mut(path) + .ok_or_else(|| anyhow::anyhow!("startup cursor missing"))?; + if let Some((sequence, event)) = next_evicted_event(daemon, cursor, range.through).await? { + ready.push((sequence, path.clone(), event)); + } + } + while !ready.is_empty() { + let index = ready + .iter() + .enumerate() + .min_by_key(|(_, (sequence, _, _))| *sequence) + .map(|(index, _)| index) + .ok_or_else(|| anyhow::anyhow!("startup lookahead missing"))?; + let (_, path, event) = ready.swap_remove(index); + dispatch_ingress_event(daemon, event).await; + let cursor = cursors + .get_mut(&path) + .ok_or_else(|| anyhow::anyhow!("startup cursor missing"))?; + if let Some((sequence, event)) = + next_evicted_event(daemon, cursor, bounds[&path].through).await? + { + ready.push((sequence, path, event)); } - anyhow::ensure!( - cursor.position == range.through, - "invalid evicted startup end" - ); } Ok(()) } +async fn next_evicted_event( + daemon: &Daemon, + cursor: &mut StartupCursor, + through: u64, +) -> anyhow::Result> { + while cursor.position < through { + let before = cursor.position; + let entry = cursor + .reader + .next_record() + .await? + .ok_or_else(|| anyhow::anyhow!("evicted startup range ends inside a record"))?; + anyhow::ensure!(entry.through <= through, "invalid evicted startup end"); + cursor.position = entry.through; + if let journal::JournalRecord::Event(redacted) = entry.record { + let mut env = journal::envelope_from_redacted(redacted); + let Some(source) = daemon.translators.canonical_source(&env.source) else { + continue; + }; + env.source = source.to_owned(); + let sequence = entry + .capture_sequence + .ok_or_else(|| anyhow::anyhow!("live startup event has no capture sequence"))?; + return Ok(Some(( + sequence, + PendingEvent { + env, + replay_through: before, + journal_through: entry.through, + settings_through: None, + }, + ))); + } + } + anyhow::ensure!(cursor.position == through, "invalid evicted startup end"); + Ok(None) +} + pub(super) enum IngressMsg { Event(Box), Barrier(oneshot::Sender<()>), @@ -386,6 +331,13 @@ pub(super) async fn recover_ingress_overflow(daemon: &Arc) { else { continue; }; + let mut start = 0; + if let Ok(Some((reader, position, _))) = resume_discovery(daemon, &path, recorded_len).await + { + checkpoints = reader; + start = position; + } + let decoder = checkpoints.decoder_context().to_vec(); let mut acknowledged_by_route: HashMap = HashMap::new(); while let Ok(Some(entry)) = checkpoints.next_record().await { if let journal::JournalRecord::DeliveryCheckpoint { route, through } = entry.record { @@ -395,10 +347,20 @@ pub(super) async fn recover_ingress_overflow(daemon: &Arc) { } } - let Ok(Some(mut reader)) = journal::JournalReader::open(&path, recorded_len).await else { - continue; + let mut reader = if start > 0 { + let Ok(reader) = + journal::JournalReader::resume(&path, start, recorded_len, decoder).await + else { + continue; + }; + reader + } else { + let Ok(Some(reader)) = journal::JournalReader::open(&path, recorded_len).await else { + continue; + }; + reader }; - let mut before = 0u64; + let mut before = start; while let Ok(Some(entry)) = reader.next_record().await { let through = entry.through; let journal::JournalRecord::Event(redacted) = entry.record else { diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 74057603..0ddae534 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -285,6 +285,9 @@ pub(super) fn spawn_session_reaper(daemon: Arc, idle_timeout: Duration) pub(super) fn spawn_gc(daemon: Arc) { tokio::spawn(async move { + // Retention may scan old transcripts. Start it after recovery releases + // live ingress, so it cannot delay resuming an active source session. + collect_garbage(&daemon).await; loop { tokio::select! { _ = daemon.shutdown.notified() => return, diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index 6320a630..93d7184f 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -45,7 +45,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; @@ -135,6 +135,7 @@ pub struct Daemon { /// Recovery must not absorb queued live events into its historical head. startup_cutoffs: Mutex>>, startup_buffer: Mutex>, + capture_sequence: AtomicU64, started: Instant, last_activity: Mutex, /// Prevents a shutdown drain from racing capture that has passed the @@ -188,6 +189,7 @@ impl Daemon { ingress_dispatched: Mutex::new(HashMap::new()), startup_cutoffs: Mutex::new(startup_cutoffs), startup_buffer: Mutex::new(startup_buffer), + capture_sequence: AtomicU64::new(0), started: Instant::now(), last_activity: Mutex::new(Instant::now()), capture_gate: tokio::sync::RwLock::new(()), @@ -272,7 +274,6 @@ pub async fn run(args: ServeArgs, opts: ServeOptions) -> anyhow::Result<()> { restore_active_parent_snapshots(&recovery_daemon.data_dir, &recovery_daemon.correlation) .await; restore_pending_sessions(&recovery_daemon).await; - collect_garbage(&recovery_daemon).await; let candidates = discover_unprocessed_journals(&recovery_daemon).await; *recovery_daemon.startup_cutoffs.lock() = None; for event in candidates { diff --git a/bt-daemon/src/server/recovery.rs b/bt-daemon/src/server/recovery.rs index 11c3b0e5..8aa35c8f 100644 --- a/bt-daemon/src/server/recovery.rs +++ b/bt-daemon/src/server/recovery.rs @@ -2,6 +2,69 @@ use super::*; +/// Checkpoint templates replace the completed prefix scan. Legacy journals +/// and incompatible checkpoints retain the full-log recovery path. +pub(super) async fn resume_discovery( + daemon: &Daemon, + path: &std::path::Path, + through: u64, +) -> anyhow::Result< + Option<( + journal::JournalReader, + u64, + Vec, + )>, +> { + let Some((_, id)) = path + .file_stem() + .and_then(|name| name.to_str()) + .and_then(|name| name.rsplit_once("--")) + else { + return Ok(None); + }; + if uuid::Uuid::parse_str(id).is_err() { + return Ok(None); + } + let storage = daemon + .data_dir + .join("pipeline") + .join(format!("{id}.sqlite")); + if !tokio::fs::try_exists(&storage).await? { + return Ok(None); + } + let partition = crate::pipeline::journal::SpanJournal::open(&storage).await?; + let Some(checkpoint) = partition.translation_checkpoint().await? else { + return Ok(None); + }; + let head = through.min(partition.translated_through().await?); + let version = daemon.version.clone(); + let input = checkpoint.input; + let context = tokio::task::spawn_blocking(move || { + crate::pipeline::translation::RecoveryContext::decode(&checkpoint, &version, head) + }) + .await??; + let Some(env) = context.templates.values().next() else { + return Ok(None); + }; + anyhow::ensure!( + path == journal::source_journal_path(&daemon.data_dir, &env.source, &env.session_id), + "checkpoint does not belong to this journal" + ); + anyhow::ensure!( + context + .templates + .values() + .all(|template| template.source == env.source && template.session_id == env.session_id), + "checkpoint contains another source session" + ); + let reader = journal::JournalReader::resume(path, input, through, context.decoder).await?; + Ok(Some(( + reader, + input, + context.templates.into_values().collect(), + ))) +} + pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec { let pending: Vec = daemon .pending_sessions @@ -31,6 +94,31 @@ pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec

= HashMap::new(); let mut latest_by_route: HashMap = HashMap::new(); + match resume_discovery(daemon, &path, recorded_len).await { + Ok(Some((resumed, input, templates))) => { + reader = resumed; + before = input; + for template in templates { + let env = journal::envelope_from_redacted(template); + daemon.remember_claude_transcripts(&env); + if let Some(route) = &env.route { + latest_by_route.insert( + serde_json::to_string(route).unwrap_or_default(), + PendingEvent { + env, + replay_through: input, + journal_through: input, + settings_through: Some(input), + }, + ); + } + } + } + Ok(None) => {} + Err(error) => { + tracing::warn!(%error, path=%path.display(), "source discovery checkpoint unavailable; scanning journal") + } + } while let Ok(Some(entry)) = reader.next_record().await { let through = entry.through; match entry.record { @@ -245,6 +333,67 @@ mod tests { } } + #[tokio::test] + async fn startup_discovery_uses_the_session_checkpoint_instead_of_scanning_history() { + use crate::pipeline::translation::{Translation, TranslationOptions}; + let tmp = tempfile::tempdir().unwrap(); + let raw = journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&raw).await.unwrap(), + )); + let mut env: Envelope = serde_json::from_value(serde_json::json!({ + "source":"debug", "session_id":"checkpoint", "event":"old", "ts_ms":1, + "payload":{}, "route":SessionRoute::default(), + })) + .unwrap(); + let mut through = 0; + for i in 0..64 { + env.ts_ms = i; + through = writer.lock().await.append(&env).await.unwrap(); + } + let translation = Translation::spawn(TranslationOptions { + source: "debug".into(), + session_id: "checkpoint".into(), + namespace: crate::ids::session_namespace("debug", "checkpoint"), + version: "test".into(), + data_dir: tmp.path().into(), + events: writer.clone(), + event_path: raw.clone(), + registry: Arc::new(Registry::default_agents()), + correlation: Arc::default(), + correlation_changed: Arc::default(), + }) + .await + .unwrap(); + translation.advance(through).await.unwrap(); + translation.barrier().await.unwrap(); + translation.shutdown().await; + let mut contents = tokio::fs::read(&raw).await.unwrap(); + for byte in &mut contents[..through as usize] { + if *byte != b'\n' { + *byte = b'x'; + } + } + tokio::fs::write(&raw, contents).await.unwrap(); + env.event = "after".into(); + let tail = writer.lock().await.append(&env).await.unwrap(); + let daemon = Daemon::new(debug_serve_options("test", tmp.path()), tmp.path().into()); + let (_, start, _) = resume_discovery(&daemon, &raw, tail) + .await + .unwrap() + .unwrap(); + assert_eq!(start, through); + let candidates = discover_unprocessed_journals(&daemon).await; + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].env.event, "after"); + assert_eq!(candidates[0].journal_through, tail); + super::super::ingress::recover_ingress_overflow(&daemon).await; + let source = daemon.translations.lock().values().next().unwrap().clone(); + source.barrier().await.unwrap(); + assert!(source.error.lock().is_none()); + super::super::maintenance::drain_all(&daemon).await; + } + #[tokio::test] async fn pending_capture_does_not_suppress_an_unrelated_durable_head() { let tmp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index 1d56b0bc..09156367 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -282,7 +282,10 @@ impl Daemon { let path = journal::source_journal_path(&self.data_dir, &env.source, &env.session_id); cutoffs.entry(path).or_insert(before); } - let through = writer.append(env).await?; + // Related hooks cannot begin until their predecessor's durable append + // is acknowledged. Concurrent sessions need no stronger total order. + let sequence = self.capture_sequence.fetch_add(1, Ordering::Relaxed); + let through = writer.append_ordered(env, Some(sequence)).await?; Ok((before, through)) } diff --git a/bt-daemon/src/translate/antigravity.rs b/bt-daemon/src/translate/antigravity.rs index d40d5a24..65305823 100644 --- a/bt-daemon/src/translate/antigravity.rs +++ b/bt-daemon/src/translate/antigravity.rs @@ -27,6 +27,7 @@ impl TranslatorFactory for crate::agents::Antigravity { } } +#[derive(serde::Serialize, serde::Deserialize)] struct Turn { span_id: String, number: u32, @@ -34,6 +35,7 @@ struct Turn { last_output: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct Invocation { span_id: String, parent_span_id: String, @@ -41,12 +43,14 @@ struct Invocation { record_start: usize, } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingTool { span_id: String, parent_span_id: String, name: String, } +#[derive(serde::Serialize, serde::Deserialize)] struct AntigravityTranslator { session_id: String, session_span_id: String, @@ -63,6 +67,7 @@ struct AntigravityTranslator { invocations: HashMap, tools: HashMap, last_ts_ms: i64, + #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, } @@ -494,6 +499,7 @@ impl AntigravityTranslator { } impl AgentTranslator for AntigravityTranslator { + translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); let mut ops = Vec::new(); diff --git a/bt-daemon/src/translate/checkpoint.rs b/bt-daemon/src/translate/checkpoint.rs new file mode 100644 index 00000000..8e60c2ae --- /dev/null +++ b/bt-daemon/src/translate/checkpoint.rs @@ -0,0 +1,67 @@ +//! Checkpoint-only SpanRow codec. The public row JSON intentionally omits +//! internal correlation and merge fields; reducer checkpoints must retain them. +use super::{LateMergeVersion, SpanRow}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +struct Row { + value: SpanRow, + late_merge_key: Option, + late_merge_version: Option, + turn_span_id: Option, + is_turn: bool, +} +impl Row { + fn from_row(value: &SpanRow) -> Self { + Self { + value: value.clone(), + late_merge_key: value.late_merge_key.clone(), + late_merge_version: value.late_merge_version.clone(), + turn_span_id: value.turn_span_id.clone(), + is_turn: value.is_turn, + } + } + fn into_row(mut self) -> SpanRow { + self.value.late_merge_key = self.late_merge_key; + self.value.late_merge_version = self.late_merge_version; + self.value.turn_span_id = self.turn_span_id; + self.value.is_turn = self.is_turn; + self.value + } +} +pub(super) mod row { + use super::*; + pub fn serialize( + value: &SpanRow, + serializer: S, + ) -> Result { + Row::from_row(value).serialize(serializer) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> Result { + Ok(Row::deserialize(deserializer)?.into_row()) + } +} +pub(super) mod rows { + use super::*; + use std::collections::BTreeMap; + pub fn serialize( + value: &BTreeMap, + serializer: S, + ) -> Result { + value + .iter() + .map(|(id, row)| (id, Row::from_row(row))) + .collect::>() + .serialize(serializer) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + Ok(BTreeMap::::deserialize(deserializer)? + .into_iter() + .map(|(id, row)| (id, row.into_row())) + .collect()) + } +} diff --git a/bt-daemon/src/translate/claude.rs b/bt-daemon/src/translate/claude.rs index 2fdcb217..2a2ca8de 100644 --- a/bt-daemon/src/translate/claude.rs +++ b/bt-daemon/src/translate/claude.rs @@ -140,6 +140,7 @@ impl TranslatorFactory for crate::agents::Claude { } } +#[derive(serde::Serialize, serde::Deserialize)] struct Turn { id: String, number: u32, @@ -149,29 +150,32 @@ struct Turn { effort: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingPrompt { turn_id: Arc, content: Value, } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct TranscriptCursor { offset: u64, buffered: Vec, } +#[derive(serde::Serialize, serde::Deserialize)] struct Subagent { span_id: String, parent_span_id: String, transcript_path: Option, } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct AgentOrigin { parent_id: String, user_turn_id: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingTool { span_id: String, parent_id: String, @@ -179,18 +183,20 @@ struct PendingTool { metadata: Value, } +#[derive(serde::Serialize, serde::Deserialize)] enum PendingHistory { Main, Owned(MessageHistory), } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct MessageHistory { messages: Vec, preserved_after_compaction: Vec, fork_context_prefix: bool, } +#[derive(serde::Serialize, serde::Deserialize)] struct HistoryMessage { value: Value, source_uuids: Vec, @@ -256,6 +262,7 @@ impl MessageHistory { } } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingEmission { segments: VecDeque>, history: PendingHistory, @@ -265,6 +272,7 @@ struct PendingEmission { clear_after: bool, } +#[derive(serde::Serialize, serde::Deserialize)] struct ClaudeTranslator { session_id: String, session_span_id: String, @@ -306,6 +314,7 @@ struct ClaudeTranslator { permission_mode: Option, effort: Option, system_prompt: Option, + #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, current_cwd: Option, last_turn_cwd: Option, @@ -1369,6 +1378,7 @@ impl ClaudeTranslator { } impl AgentTranslator for ClaudeTranslator { + translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); anyhow::ensure!( diff --git a/bt-daemon/src/translate/codex.rs b/bt-daemon/src/translate/codex.rs index f6731819..be91fcdd 100644 --- a/bt-daemon/src/translate/codex.rs +++ b/bt-daemon/src/translate/codex.rs @@ -139,12 +139,13 @@ impl TranslatorFactory for crate::agents::Codex { } } -#[derive(PartialEq, Eq, Clone, Copy)] +#[derive(PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)] enum ScopeKind { Main, Subagent, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenTurn { turn_id: String, span_id: String, @@ -156,12 +157,13 @@ struct OpenTurn { native_prompt_seen: bool, } -#[derive(PartialEq, Eq, Clone, Copy)] +#[derive(PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)] enum TurnInputSource { Native, Authoritative, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenLlm { span_id: String, turn_id: String, @@ -171,6 +173,7 @@ struct OpenLlm { output_preset: bool, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenTool { span_id: String, turn_id: String, @@ -178,6 +181,7 @@ struct OpenTool { name: String, } +#[derive(serde::Serialize, serde::Deserialize)] struct Scope { path: String, kind: ScopeKind, @@ -212,6 +216,7 @@ impl Scope { } } +#[derive(serde::Serialize, serde::Deserialize)] enum DeferredHook { None, PostToolUse(Value), @@ -230,6 +235,7 @@ enum DeferredHook { }, } +#[derive(serde::Serialize, serde::Deserialize)] enum PendingWork { Hook { path: String, @@ -246,6 +252,7 @@ enum PendingWork { }, } +#[derive(serde::Serialize, serde::Deserialize)] struct CodexTranslator { session_id: String, /// The daemon-owned session row. Child spans always parent to this id. @@ -267,10 +274,12 @@ struct CodexTranslator { compaction_trigger_by_turn: RecentMap, compaction_spans: RecentSet, pending: Option, + #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, } impl AgentTranslator for CodexTranslator { + translator_checkpoint!(); fn handle(&mut self, event: &Envelope, _ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), diff --git a/bt-daemon/src/translate/cursor/mod.rs b/bt-daemon/src/translate/cursor/mod.rs index 18b59369..e8b26958 100644 --- a/bt-daemon/src/translate/cursor/mod.rs +++ b/bt-daemon/src/translate/cursor/mod.rs @@ -73,7 +73,7 @@ impl TranslatorFactory for crate::agents::Cursor { } } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct History { messages: VecDeque, bytes: usize, @@ -100,7 +100,7 @@ impl History { } } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct Turn { ordinal: u64, id: String, @@ -113,7 +113,9 @@ struct Turn { emitted_text: String, usage: serde_json::Map, } +#[derive(serde::Serialize, serde::Deserialize)] struct ModelStep { + #[serde(with = "crate::translate::checkpoint::row")] row: SpanRow, content: Vec, text: String, @@ -153,14 +155,15 @@ impl ModelStep { self.content.push(value); } } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct Tool { + #[serde(with = "crate::translate::checkpoint::row")] row: SpanRow, turn: String, call_id: String, ended: bool, } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct TranscriptCursor { path: String, offset: u64, @@ -208,6 +211,7 @@ impl TranscriptCursor { } } +#[derive(serde::Serialize, serde::Deserialize)] struct CursorTranslator { namespace: String, root: String, @@ -218,6 +222,7 @@ struct CursorTranslator { cwd: Option, model: Option, model_params: Option, + #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, turn_seq: u64, turn: Option, @@ -233,6 +238,7 @@ struct CursorTranslator { transcript_owner: Option, transcript_unattributed: bool, pending: Option, + #[serde(with = "crate::translate::checkpoint::rows")] subagents: BTreeMap, compact_seq: u64, } @@ -1535,6 +1541,7 @@ impl CursorTranslator { } } impl AgentTranslator for CursorTranslator { + translator_checkpoint!(); fn handle(&mut self, e: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), diff --git a/bt-daemon/src/translate/debug.rs b/bt-daemon/src/translate/debug.rs index 6af1a0b5..5e9f2555 100644 --- a/bt-daemon/src/translate/debug.rs +++ b/bt-daemon/src/translate/debug.rs @@ -36,6 +36,7 @@ impl TranslatorFactory for DebugTranslatorFactory { } } +#[derive(serde::Serialize, serde::Deserialize)] struct DebugTranslator { root_span_id: String, root_emitted: bool, @@ -43,6 +44,7 @@ struct DebugTranslator { } impl AgentTranslator for DebugTranslator { + translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { let mut ops = Vec::new(); diff --git a/bt-daemon/src/translate/grok.rs b/bt-daemon/src/translate/grok.rs index d474844c..cc9adbd7 100644 --- a/bt-daemon/src/translate/grok.rs +++ b/bt-daemon/src/translate/grok.rs @@ -32,7 +32,7 @@ impl TranslatorFactory for crate::agents::Grok { } } -#[derive(Clone, Default)] +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] struct BoundedOutput { values: Vec, bytes: usize, @@ -111,6 +111,7 @@ impl BoundedOutput { } } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenTurn { span_id: String, key: String, @@ -128,6 +129,7 @@ struct TurnCompletion { late_merge_key: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenLlm { span_id: String, parent_span_id: String, @@ -139,12 +141,14 @@ struct OpenLlm { last_ms: i64, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenTool { span_id: String, parent_span_id: String, start_ms: i64, } +#[derive(serde::Serialize, serde::Deserialize)] struct CompletedTool { span_id: String, parent_span_id: String, @@ -154,7 +158,7 @@ struct CompletedTool { failure_recorded: bool, } -#[derive(Clone, Default)] +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] struct TranscriptCursor { path: Option, offset: u64, @@ -192,11 +196,12 @@ struct TranscriptBatch { reset: bool, } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct PendingWork { event: Envelope, } +#[derive(serde::Serialize, serde::Deserialize)] struct GrokTranslator { session_id: String, session_span_id: String, @@ -221,6 +226,7 @@ struct GrokTranslator { first_llm_user_input: Option, pending: Option, cwd: Option, + #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, last_ts_ms: i64, } @@ -1222,6 +1228,7 @@ impl GrokTranslator { } impl AgentTranslator for GrokTranslator { + translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), diff --git a/bt-daemon/src/translate/mod.rs b/bt-daemon/src/translate/mod.rs index 40780482..78c29ada 100644 --- a/bt-daemon/src/translate/mod.rs +++ b/bt-daemon/src/translate/mod.rs @@ -7,10 +7,25 @@ //! whole pipeline be exercised with a debug sink and makes translators unit- //! testable without any network. +// Checkpoint codecs stay local to each concrete reducer; the producer owns +// format compatibility, compression, and the atomic SQLite input boundary. +macro_rules! translator_checkpoint { + () => { + fn save_state(&self) -> anyhow::Result>> { + Ok(Some(serde_json::to_vec(self)?)) + } + fn restore_state(&mut self, state: &[u8]) -> anyhow::Result<()> { + *self = serde_json::from_slice(state)?; + Ok(()) + } + }; +} + mod shared; pub use shared::CowSpanOp; mod antigravity; +mod checkpoint; mod claude; mod codex; pub(crate) mod cursor; @@ -148,6 +163,15 @@ pub struct SessionCtx { /// A per-session state machine. One instance per session; `&mut self` so it /// can hold open-span maps, transcript offsets, etc. pub trait AgentTranslator: Send { + /// Reducer context after a completely drained source event. These bytes + /// are versioned by the producer and committed alongside its input head. + fn save_state(&self) -> anyhow::Result>> { + Ok(None) + } + fn restore_state(&mut self, _state: &[u8]) -> anyhow::Result<()> { + anyhow::bail!("translator does not support checkpoints") + } + /// Handle one event, returning span ops to emit. fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result>; diff --git a/bt-daemon/src/translate/muse.rs b/bt-daemon/src/translate/muse.rs index ff757aee..7fddd4e9 100644 --- a/bt-daemon/src/translate/muse.rs +++ b/bt-daemon/src/translate/muse.rs @@ -26,12 +26,13 @@ impl TranslatorFactory for crate::agents::Muse { Box::new(MuseTranslator::new(session_id, GitMetadataCache::shared())) } } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct OpenOp { id: String, parents: Vec, turn_span_id: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct MuseTranslator { session_id: String, root_id: String, @@ -49,6 +50,7 @@ struct MuseTranslator { usage: UsageTracker, last_completed_turn: Option, last_ts: i64, + #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, } impl MuseTranslator { @@ -329,6 +331,7 @@ impl MuseTranslator { } } impl AgentTranslator for MuseTranslator { + translator_checkpoint!(); fn handle(&mut self, e: &Envelope, ctx: &SessionCtx) -> Result> { self.last_ts = self.last_ts.max(e.ts_ms); let event = match MuseEvent::decode(e) { diff --git a/bt-daemon/src/translate/muse_events.rs b/bt-daemon/src/translate/muse_events.rs index 2e3de370..93a46306 100644 --- a/bt-daemon/src/translate/muse_events.rs +++ b/bt-daemon/src/translate/muse_events.rs @@ -1,6 +1,6 @@ //! Typed tool, observation, subagent, and compaction reducers. use super::*; -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] pub(super) struct Events { open: HashMap, closed: RecentSet, diff --git a/bt-daemon/src/translate/muse_usage.rs b/bt-daemon/src/translate/muse_usage.rs index 4a889322..98b20044 100644 --- a/bt-daemon/src/translate/muse_usage.rs +++ b/bt-daemon/src/translate/muse_usage.rs @@ -3,7 +3,7 @@ use super::super::recent::RecentMap; use crate::muse::usage::{TokenMetrics, TokenUsage, UsageScope}; -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] pub(super) struct UsageTracker { turn_usage: RecentMap, model_metrics: RecentMap, diff --git a/bt-daemon/src/translate/opencode.rs b/bt-daemon/src/translate/opencode.rs index c487b0cf..079baab0 100644 --- a/bt-daemon/src/translate/opencode.rs +++ b/bt-daemon/src/translate/opencode.rs @@ -340,7 +340,7 @@ fn properties(payload: &Value, field: &str) -> Option { .and_then(decode) } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct NativeSession { root_span_id: String, effective_root_span_id: String, @@ -374,6 +374,7 @@ struct NativeSession { history_tool_results: HashMap>, } +#[derive(serde::Serialize, serde::Deserialize)] struct ClosedToolSpan { span_id: String, root_span_id: String, @@ -389,17 +390,19 @@ impl NativeSession { } } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct MessageHistory { entries: Vec, pending_compaction: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct HistoryEntry { message_id: String, values: Vec, } +#[derive(serde::Serialize, serde::Deserialize)] struct CompactionBoundary { tail_start_id: Option, } @@ -493,7 +496,7 @@ impl MessageHistory { } } -#[derive(Clone, Default)] +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] struct PermissionRequest { id: Option, session_id: Option, @@ -504,15 +507,18 @@ struct PermissionRequest { permission_type: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenCodeTranslator { daemon_session_id: String, sessions: HashMap, permission_requests: HashMap, + #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, last_ts_ms: i64, } impl AgentTranslator for OpenCodeTranslator { + translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); let mut ops = match event.event.as_str() { diff --git a/bt-daemon/src/translate/pi.rs b/bt-daemon/src/translate/pi.rs index 477df5a0..93e90746 100644 --- a/bt-daemon/src/translate/pi.rs +++ b/bt-daemon/src/translate/pi.rs @@ -61,7 +61,7 @@ struct BeforeAgentStart { /// `ctx.model`: the selected model, which may be a Pi 1.0 virtual model that /// routes each request to a physical model. -#[derive(Deserialize)] +#[derive(Deserialize, serde::Serialize)] struct SelectedModel { provider: String, id: String, @@ -309,6 +309,7 @@ fn decode(value: &Value) -> Option { serde_json::from_value(value.clone()).ok() } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingLlm { start_ms: i64, input: Value, @@ -316,13 +317,14 @@ struct PendingLlm { request_config: Map, selected_model: Option, } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct ToolStart { start_ms: i64, name: String, args: Value, parent_call: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct PiTranslator { session_id: String, root_span_id: String, @@ -349,10 +351,12 @@ struct PiTranslator { branch_summary: Option<(String, i64, Value)>, last_ts: i64, thinking_level: Option, + #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, } impl AgentTranslator for PiTranslator { + translator_checkpoint!(); fn handle(&mut self, envelope: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts = self.last_ts.max(envelope.ts_ms); let event = envelope.payload.get("event").unwrap_or(&envelope.payload); diff --git a/bt-daemon/src/translate/recent.rs b/bt-daemon/src/translate/recent.rs index 557280be..4b1412ec 100644 --- a/bt-daemon/src/translate/recent.rs +++ b/bt-daemon/src/translate/recent.rs @@ -11,6 +11,8 @@ use std::hash::Hash; pub(super) const RECENT_ID_CAPACITY: usize = 1_024; +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(bound(deserialize = "K: Eq + Hash + serde::Deserialize<'de>"))] pub(super) struct RecentSet { values: HashSet, order: VecDeque, @@ -59,6 +61,8 @@ impl RecentSet { } } +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(bound(deserialize = "K: Eq + Hash + serde::Deserialize<'de>, V: serde::Deserialize<'de>"))] pub(super) struct RecentMap { values: HashMap, order: VecDeque, diff --git a/bt-daemon/src/translate/tool.rs b/bt-daemon/src/translate/tool.rs index aabe0ae6..49573345 100644 --- a/bt-daemon/src/translate/tool.rs +++ b/bt-daemon/src/translate/tool.rs @@ -6,7 +6,7 @@ use serde_json::{json, Map, Value}; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum ToolApproval { Approved, Denied, diff --git a/bt-daemon/src/translate/turn_lineage.rs b/bt-daemon/src/translate/turn_lineage.rs index 50d466b3..4006d227 100644 --- a/bt-daemon/src/translate/turn_lineage.rs +++ b/bt-daemon/src/translate/turn_lineage.rs @@ -45,6 +45,13 @@ fn publish_turn(row: &mut SpanRow) { } impl AgentTranslator for TurnLineage { + fn save_state(&self) -> anyhow::Result>> { + self.inner.save_state() + } + fn restore_state(&mut self, state: &[u8]) -> anyhow::Result<()> { + self.inner.restore_state(state) + } + fn handle( &mut self, event: &crate::wire::Envelope, diff --git a/bt-daemon/tests/translator_checkpoints.rs b/bt-daemon/tests/translator_checkpoints.rs new file mode 100644 index 00000000..669756a0 --- /dev/null +++ b/bt-daemon/tests/translator_checkpoints.rs @@ -0,0 +1,202 @@ +#![allow( + clippy::unwrap_used, + clippy::expect_used, + reason = "Checkpoint regression fixtures fail immediately." +)] +#[path = "support/distributed.rs"] +#[allow(dead_code)] +mod distributed; +use bt_daemon::wire::Envelope; +use bt_daemon::{AgentTranslator, Registry, SessionCtx, SpanOp}; +use distributed::{AgentKind, DistributedFixtures, ProcessTree}; +use serde_json::{json, Value}; + +fn event(source: &str, name: &str, ts: i64, payload: Value) -> Envelope { + Envelope { + source: source.into(), + source_version: None, + plugin_version: None, + session_id: "checkpoint".into(), + event: name.into(), + ts_ms: ts, + managed_run_id: None, + capture: None, + payload, + route: None, + config: None, + } +} +fn collect(translator: &mut dyn AgentTranslator, env: &Envelope, ctx: &SessionCtx) -> Vec { + let mut ops = translator.handle(env, ctx).unwrap(); + while let Some(batch) = translator.drain_pending(ctx).unwrap() { + ops.extend(batch); + } + ops +} +fn comparable(ops: Vec) -> Vec { + let mut values = ops + .into_iter() + .map(|op| { + let (SpanOp::Insert(row) | SpanOp::Merge(row)) = &op; + serde_json::to_string(&( + &op, + &row.turn_span_id, + row.is_turn, + &row.late_merge_key, + &row.late_merge_version, + )) + .unwrap() + }) + .collect::>(); + values.sort(); + values +} +fn assert_stream(source: &str, events: Vec) { + let registry = Registry::default_agents(); + let session = events + .first() + .map(|env| env.session_id.as_str()) + .unwrap_or("checkpoint"); + let ctx = SessionCtx { + session_id: session.into(), + config: None, + }; + for split in 0..=events.len() { + let mut original = registry.create(source, session); + for env in &events[..split] { + collect(original.as_mut(), env, &ctx); + } + let state = original + .save_state() + .unwrap() + .expect("every translator must checkpoint"); + let mut restored = registry.create(source, session); + restored.restore_state(&state).unwrap(); + for env in &events[split..] { + assert_eq!( + comparable(collect(original.as_mut(), env, &ctx)), + comparable(collect(restored.as_mut(), env, &ctx)), + "{source} split {split} event {}", + env.event + ); + } + assert_eq!( + comparable(original.finalize(&ctx).unwrap()), + comparable(restored.finalize(&ctx).unwrap()), + "{source} split {split} finalization" + ); + } +} +#[test] +fn checkpoints_preserve_native_turns_tools_transcripts_and_later_turn_ids() { + for kind in AgentKind::ALL { + let mut fixtures = DistributedFixtures::new(); + let tree = ProcessTree::root(1234); + let mut events = fixtures.start_turn(kind, "checkpoint", &tree, "first", 1_700_000_000_000); + events.push(fixtures.open_tool( + kind, + "checkpoint", + &tree, + "call", + "work", + 1_700_000_000_010, + )); + events.push(fixtures.close_tool( + kind, + "checkpoint", + &tree, + "call", + "work", + "result", + 1_700_000_000_020, + )); + events.push(fixtures.close_session( + kind, + "checkpoint", + &tree, + "complete", + 1_700_000_000_030, + )); + events.extend(fixtures.start_turn(kind, "checkpoint", &tree, "second", 1_700_000_000_040)); + let source = events[0].source.clone(); + assert_stream(&source, events); + } +} +#[test] +fn muse_checkpoints_preserve_open_operations_and_cumulative_usage() { + let events = include_str!("fixtures/muse-hooks-1.1.1.jsonl") + .lines() + .enumerate() + .map(|(i, line)| { + let payload: Value = serde_json::from_str(line).unwrap(); + let name = payload["hook_event_name"].as_str().unwrap().to_owned(); + event("muse", &name, i as i64 + 1, payload) + }) + .collect(); + assert_stream("muse", events); +} +#[test] +fn cursor_checkpoints_preserve_open_tools_and_internal_span_fields() { + let events = [ + ("sessionStart", json!({})), + ("beforeSubmitPrompt", json!({"prompt":"work"})), + ( + "preToolUse", + json!({"tool_name":"Shell","tool_use_id":"call","tool_input":{"command":"echo hello"}}), + ), + ( + "postToolUse", + json!({"tool_name":"Shell","tool_use_id":"call","tool_output":"hello","tool_input":{"command":"echo hello"},"duration":1}), + ), + ("afterAgentResponse", json!({"text":"done"})), + ("stop", json!({"status":"completed"})), + ("beforeSubmitPrompt", json!({"prompt":"next"})), + ] + .into_iter() + .enumerate() + .map(|(i, (name, mut payload))| { + payload["hook_event_name"] = name.into(); + payload["conversation_id"] = "checkpoint".into(); + payload["generation_id"] = format!("generation-{i}").into(); + payload["model"] = "test".into(); + payload["workspace_roots"] = json!(["/workspace"]); + event("cursor", name, i as i64 + 1, payload) + }) + .collect(); + assert_stream("cursor", events); +} +#[test] +fn antigravity_checkpoints_preserve_invocations_across_native_boundaries() { + let records = [ + json!({"step_index":0,"source":"USER_EXPLICIT","type":"USER_INPUT","status":"COMPLETED","content":"List files"}), + json!({"step_index":1,"source":"MODEL","type":"PLANNER_RESPONSE","status":"COMPLETED","content":"Inspecting","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"}}],"usage":{"inputTokens":12,"outputTokens":7,"totalTokens":19}}), + json!({"step_index":2,"source":"MODEL","type":"CORTEX_STEP_TYPE_RUN_COMMAND","status":"COMPLETED","content":"README","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"},"result":"README"}]}), + ]; + let mut contents = String::new(); + let mut boundaries = Vec::new(); + for record in records { + contents.push_str(&serde_json::to_string(&record).unwrap()); + contents.push('\n'); + boundaries.push(contents.len()); + } + let events = [("PreInvocation", boundaries[0]), ("PostInvocation", boundaries[1]), ("PostToolUse", boundaries[2]), ("Stop", boundaries[2])].into_iter().enumerate().map(|(i, (name, through))| { + event("antigravity", name, i as i64 + 1, json!({"conversationId":"checkpoint","invocationNum":0,"initialNumSteps":1,"stepIdx":2,"fullyIdle":true, + "transcriptPath":"/tmp/checkpoint/transcript.jsonl", + "_bt_transcript_observation":{"path":"/tmp/checkpoint/transcript.jsonl","observed_bytes":through,"full_path":"/tmp/checkpoint/transcript_full.jsonl","full_observed_bytes":through}, + "_bt_transcript_snapshot":{"path":"/tmp/checkpoint/transcript_full.jsonl","contents":contents}})) + }).collect(); + assert_stream("antigravity", events); +} +#[test] +fn grok_checkpoints_preserve_transcript_cursors_and_pending_model_work() { + let root = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/grok/transcript"); + let updates = root.join("updates.jsonl"); + let events = root.join("events.jsonl"); + let mut offset = 0; + let stream = std::fs::read(&updates).unwrap().split_inclusive(|byte| *byte == b'\n').map(|line| { + offset += line.len() as u64; + event("grok", "Wake", 1600, json!({"cwd":"/repo","_bt_grok_transcript_mirrors":{"updates":{"mirror":updates,"through":offset},"events":{"mirror":events,"through":std::fs::metadata(&events).unwrap().len()}}})) + }).collect(); + assert_stream("grok", stream); +} From ae1a344ac82410f0f30b81318a4dc9083b7db3fd Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 19:48:42 +0800 Subject: [PATCH 05/19] Remove recovery internals from the customer README --- bt-daemon/README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bt-daemon/README.md b/bt-daemon/README.md index 311ec539..0a1c5127 100644 --- a/bt-daemon/README.md +++ b/bt-daemon/README.md @@ -58,13 +58,6 @@ or `bt trace doctor ` for configuration, authentication, and recent session diagnostics. Hook exit codes can be observed by the caller while the coding agent decides whether to continue its turn. -Hooks continue recording events while the daemon recovers earlier work. Each -source session has one translation position and a periodic context checkpoint -in its existing SQLite store; each destination has its own SpanOp delivery -position. Recovery restores open turns, tools, and transcript cursors from the -checkpoint and replays only the remaining suffix. An unavailable or incompatible -checkpoint falls back to rebuilding from the event journal. - ## Per-agent plugin settings Each coding agent reads an independent non-credential `braintrust.json` file: From 1cd41e83245d4831209cf651f12b8a7ec50c5ab8 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 19:56:57 +0800 Subject: [PATCH 06/19] Verify positional recovery for every agent journal --- bt-daemon/src/journal.rs | 7 +-- bt-daemon/src/pipeline/translation.rs | 62 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index b5751eaf..34a50c6b 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -358,12 +358,13 @@ impl JournalReader { &self.pi_context } - /// Resume at a committed record boundary with the Pi delta decoder's base. + /// Resume any source journal at a committed byte offset. Ordinary NDJSON + /// needs no decoder context; Pi's delta encoding also needs its saved base. pub(crate) async fn resume( path: &Path, position: u64, through: u64, - pi_context: Vec, + decoder_context: Vec, ) -> anyhow::Result { let mut file = tokio::fs::File::open(path).await?; anyhow::ensure!( @@ -383,7 +384,7 @@ impl JournalReader { path: path.to_owned(), line_no: 0, position, - pi_context, + pi_context: decoder_context, }) } /// Open a journal for streaming, reading at most `through` bytes. diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 7ce12a3e..0a826dfa 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -1071,6 +1071,68 @@ mod tests { assert!(checkpoint.await.unwrap().unwrap().is_some()); } + #[tokio::test] + async fn every_agent_journal_resumes_at_its_persisted_byte_offset() { + for source in Registry::default_agents().sources() { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), &source, "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let translation = + Translation::spawn(resume_options(tmp.path(), &source, writer.clone())) + .await + .unwrap(); + let control = PipelineControl { + kind: ControlKind::Checkpoint, + bounds: HashMap::new(), + }; + let mut tail = 0; + // Ordered controls exercise the real producer and each concrete + // translator without depending on a particular native hook schema. + // Native open-span/state equivalence is covered separately. + for _ in 0..CHECKPOINT_EVENTS + 3 { + tail = writer.lock().await.append_control(&control).await.unwrap(); + } + translation.advance(tail).await.unwrap(); + translation.barrier().await.unwrap(); + let checkpoint = translation + .journal + .translation_checkpoint() + .await + .unwrap() + .unwrap(); + assert!( + checkpoint.input > 0 && checkpoint.input < tail, + "{source}: periodic checkpoint must precede the uncheckpointed suffix" + ); + let task = translation.task.lock().await.take().unwrap(); + task.abort(); + let _ = task.await; + let mut contents = tokio::fs::read(&path).await.unwrap(); + for byte in &mut contents[..checkpoint.input as usize] { + if *byte != b'\n' { + *byte = b'x'; + } + } + tokio::fs::write(&path, contents).await.unwrap(); + tail = writer.lock().await.append_control(&control).await.unwrap(); + let resumed = Translation::spawn(resume_options(tmp.path(), &source, writer.clone())) + .await + .unwrap(); + resumed.advance(tail).await.unwrap(); + resumed.barrier().await.unwrap_or_else(|error| { + panic!("{source} could not resume past its saved byte offset: {error}") + }); + assert_eq!( + resumed.journal.translated_through().await.unwrap(), + tail, + "{source}" + ); + resumed.shutdown().await; + } + } + #[tokio::test] async fn checkpoint_seeks_past_completed_history_and_restores_pi_delta_context() { for source in ["debug", "pi"] { From 9ff122c91563ada57e82744f18b9d3bf1eb55298 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 20:54:37 +0800 Subject: [PATCH 07/19] Share lossless journal payload encoding across coding agents --- bt-daemon/src/journal.rs | 455 ++++++++++++-------------- bt-daemon/src/journal/codec.rs | 300 +++++++++++++++++ bt-daemon/src/pipeline/translation.rs | 149 ++++++++- bt-daemon/src/server/ingress.rs | 2 +- bt-daemon/src/server/maintenance.rs | 2 +- bt-daemon/src/translate/mod.rs | 1 - bt-daemon/src/translate/pi.rs | 34 ++ bt-daemon/tests/pipeline.rs | 19 +- 8 files changed, 703 insertions(+), 259 deletions(-) create mode 100644 bt-daemon/src/journal/codec.rs diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index 34a50c6b..9207cd69 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -10,6 +10,9 @@ //! still tell which delivery pipelines a managed child produced after the //! daemon that accepted them has restarted or idle-exited. +mod codec; +pub(crate) use codec::DecoderContext; + use crate::wire::{BackendAuth, Envelope, RedactedEnvelope, SessionRoute}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; @@ -221,7 +224,7 @@ pub async fn gc_old_managed_runs(data_dir: &Path, max_age: std::time::Duration) pub struct JournalWriter { file: tokio::fs::File, position: u64, - pi_context: Vec, + payloads: codec::PayloadCache, } impl JournalWriter { @@ -238,7 +241,7 @@ impl JournalWriter { Ok(Self { file, position, - pi_context: Vec::new(), + payloads: codec::PayloadCache::default(), }) } @@ -250,9 +253,9 @@ impl JournalWriter { /// /// The journal is a durability record and is never capped or truncated: /// dropping entries would silently cost recovery fidelity. Its size is - /// bounded instead by writing each transcript byte once (see - /// [`crate::transcript_mirror`]) and by age-based GC, and replay reads it - /// as a stream so a large journal never becomes a large allocation. + /// managed by age-based GC. Native event payloads are preserved, including + /// repeated message snapshots. External transcript mirrors capture each + /// byte once, and replay streams records without loading the whole log. #[cfg(test)] pub async fn append(&mut self, env: &Envelope) -> anyhow::Result { self.append_ordered(env, None).await @@ -267,13 +270,21 @@ impl JournalWriter { struct Captured<'a> { #[serde(flatten)] envelope: &'a RedactedEnvelope, + _bt_record_type: &'static str, #[serde(skip_serializing_if = "Option::is_none")] _bt_capture_sequence: Option, } let mut redacted = env.redacted(); - compact_pi_payload(&mut redacted, &mut self.pi_context); + let (json, delta) = self.payloads.encode(&env.event, &redacted.payload)?; + let record_type = if let Some(delta) = delta { + redacted.payload = serde_json::to_value(delta)?; + codec::DELTA_RECORD + } else { + "event" + }; let mut line = serde_json::to_vec(&Captured { envelope: &redacted, + _bt_record_type: record_type, _bt_capture_sequence: sequence, })?; line.push(b'\n'); @@ -281,6 +292,7 @@ impl JournalWriter { self.file.flush().await?; self.file.sync_data().await?; self.position += line.len() as u64; + self.payloads.record(&env.event, self.position, json); Ok(self.position) } @@ -337,6 +349,8 @@ pub struct JournalRecordEntry { pub record: JournalRecord, /// Exclusive byte offset after this record in the journal. pub through: u64, + /// Decoded work, which can be much larger than a compressed record. + pub(crate) decoded_bytes: u64, /// Daemon-local capture order. Kept outside the native event payload. pub(crate) capture_sequence: Option, } @@ -350,21 +364,21 @@ pub struct JournalReader { path: PathBuf, line_no: usize, position: u64, - pi_context: Vec, + decoder: DecoderContext, } impl JournalReader { - pub(crate) fn decoder_context(&self) -> &[serde_json::Value] { - &self.pi_context + pub(crate) fn decoder_context(&self) -> &DecoderContext { + &self.decoder } - /// Resume any source journal at a committed byte offset. Ordinary NDJSON - /// needs no decoder context; Pi's delta encoding also needs its saved base. + /// Resume any source journal at a committed byte offset with its saved + /// lossless payload cache (and, for old Pi logs, their legacy message base). pub(crate) async fn resume( path: &Path, position: u64, through: u64, - decoder_context: Vec, + decoder_context: DecoderContext, ) -> anyhow::Result { let mut file = tokio::fs::File::open(path).await?; anyhow::ensure!( @@ -384,7 +398,7 @@ impl JournalReader { path: path.to_owned(), line_no: 0, position, - pi_context: decoder_context, + decoder: decoder_context, }) } /// Open a journal for streaming, reading at most `through` bytes. @@ -405,7 +419,7 @@ impl JournalReader { path: path.to_path_buf(), line_no: 0, position: 0, - pi_context: Vec::new(), + decoder: DecoderContext::default(), })) } @@ -453,6 +467,7 @@ impl JournalReader { let capture_sequence = value .get("_bt_capture_sequence") .and_then(serde_json::Value::as_u64); + let mut decoded_bytes = read as u64; let record = if value .get("_bt_record_type") .and_then(serde_json::Value::as_str) @@ -473,17 +488,50 @@ impl JournalReader { { JournalRecord::Control(serde_json::from_value(value)?) } else { - let mut event = serde_json::from_value(value).map_err(|error| { - anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) - })?; - expand_pi_payload(&mut event, &mut self.pi_context).map_err(|error| { - anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) - })?; + let record_type = value + .get("_bt_record_type") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let mut event: RedactedEnvelope = + serde_json::from_value(value).map_err(|error| { + anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) + })?; + match record_type.as_deref() { + Some("event") | Some(codec::DELTA_RECORD) => { + self.decoder.legacy_pi.clear(); + let json = if record_type.as_deref() == Some(codec::DELTA_RECORD) { + let (payload, json) = + self.decoder.payloads.decode(&event.event, event.payload)?; + decoded_bytes = decoded_bytes.max(json.len() as u64); + event.payload = payload; + Some(json) + } else { + codec::cache_json(&event.event, &event.payload)? + }; + self.decoder + .payloads + .record(&event.event, self.position, json); + } + None => { + self.decoder.payloads = codec::PayloadCache::default(); + expand_legacy_pi_payload(&mut event, &mut self.decoder.legacy_pi).map_err( + |error| { + anyhow::anyhow!( + "journal {}:{}: {error}", + self.path.display(), + self.line_no + ) + }, + )?; + } + Some(kind) => anyhow::bail!("unsupported journal record type {kind:?}"), + } JournalRecord::Event(event) }; return Ok(Some(JournalRecordEntry { record, through: self.position, + decoded_bytes, capture_sequence, })); } @@ -519,71 +567,11 @@ impl JournalReader { } } -const PI_MESSAGES_DELTA: &str = "_bt_messages_delta"; +const LEGACY_PI_MESSAGES_DELTA: &str = "_bt_messages_delta"; -fn compact_pi_payload(event: &mut RedactedEnvelope, previous: &mut Vec) { - if event.source != "pi" { - return; - } - let Some(native) = event - .payload - .get_mut("event") - .and_then(serde_json::Value::as_object_mut) - else { - return; - }; - match event.event.as_str() { - "context" => { - let Some(messages) = native - .remove("messages") - .and_then(|messages| messages.as_array().cloned()) - else { - return; - }; - let common_prefix = previous - .iter() - .zip(&messages) - .take_while(|(left, right)| left == right) - .count(); - native.insert( - PI_MESSAGES_DELTA.into(), - serde_json::json!({ - "common_prefix": common_prefix, - "suffix": messages[common_prefix..], - }), - ); - *previous = messages; - } - "before_provider_request" => { - // The translator records only the allowlisted request - // configuration and tool definitions, never the prompt or messages. - if let Some(payload) = native.get_mut("payload") { - *payload = serde_json::Value::Object(crate::translate::pi_request_config(payload)); - } - } - "agent_end" => { - // The translator only consumes willRetry from this lifecycle event. - native.remove("messages"); - } - "message_update" => { - // Pi repeats the progressively growing assistant message on every - // streaming update. Translation only needs the update type to mark - // time-to-first-token; the completed message arrives separately in - // message_end. - native.remove("message"); - if let Some(update) = native - .get_mut("assistantMessageEvent") - .and_then(serde_json::Value::as_object_mut) - { - update.retain(|key, _| key == "type"); - } - native.retain(|key, _| key == "type" || key == "assistantMessageEvent"); - } - _ => {} - } -} - -fn expand_pi_payload( +// Read compatibility for journals written before native payload preservation. +// New event records are tagged explicitly and never enter this decoder. +fn expand_legacy_pi_payload( event: &mut RedactedEnvelope, previous: &mut Vec, ) -> anyhow::Result<()> { @@ -597,7 +585,7 @@ fn expand_pi_payload( else { return Ok(()); }; - let Some(delta) = native.remove(PI_MESSAGES_DELTA) else { + let Some(delta) = native.remove(LEGACY_PI_MESSAGES_DELTA) else { if let Some(messages) = native.get("messages").and_then(serde_json::Value::as_array) { *previous = messages.clone(); } @@ -763,185 +751,160 @@ mod tests { } #[tokio::test] - async fn pi_context_history_is_delta_encoded_and_replayed() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("pi.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let mut messages = Vec::new(); - let mut naive_bytes = 0usize; - for index in 0..20 { - messages.push(serde_json::json!({ - "role":"user", - "content": format!("{index}:{}", "x".repeat(4096)), - })); - let event = pi_context(messages.clone(), index); - naive_bytes += serde_json::to_vec(&event.redacted()).unwrap().len() + 1; - writer.append(&event).await.unwrap(); + async fn every_source_journals_complete_payloads_in_event_order() { + for source in crate::translate::Registry::default_agents().sources() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let mut expected = Vec::new(); + for (index, name) in [ + "context", + "before_provider_request", + "message_update", + "agent_end", + ] + .into_iter() + .enumerate() + { + let mut event = pi_context( + vec![serde_json::json!({"role":"user","content":"entire history"})], + index as i64, + ); + event.source = source.clone(); + event.event = name.into(); + event.payload["event"]["type"] = name.into(); + event.payload["event"]["message"] = + serde_json::json!({"role":"assistant","content":"growing message"}); + event.payload["event"]["assistantMessageEvent"] = serde_json::json!({"type":"text_delta","delta":"token","partial":"partial message"}); + event.payload["event"]["payload"] = serde_json::json!({"system":"system prompt","messages":[{"role":"user","content":"provider messages"}],"unknown_provider_field":{"preserve":true}}); + writer.append(&event).await.unwrap(); + expected.push(event); + } + drop(writer); + let stored = tokio::fs::read_to_string(&path).await.unwrap(); + let records = stored + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + let mut reader = JournalReader::open(&path, stored.len() as u64) + .await + .unwrap() + .unwrap(); + for (index, event) in expected.iter().enumerate() { + assert_eq!( + records[index]["payload"], event.payload, + "{source}: {} was changed before persistence", + event.event + ); + let replayed = reader.next_entry().await.unwrap().unwrap(); + assert_eq!(replayed.event, event.event); + assert_eq!(replayed.payload, event.payload); + assert!( + reader.decoder_context().legacy_pi.is_empty(), + "new {source} records must not require the legacy Pi message decoder" + ); + } + assert!(reader.next_entry().await.unwrap().is_none()); } - let compacted = vec![ - serde_json::json!({"role":"compactionSummary","summary":"bounded"}), - serde_json::json!({"role":"user","content":"after"}), - ]; - let event = pi_context(compacted.clone(), 20); - naive_bytes += serde_json::to_vec(&event.redacted()).unwrap().len() + 1; - writer.append(&event).await.unwrap(); - drop(writer); - - let stored = tokio::fs::read(&path).await.unwrap(); - assert!( - stored.len() < naive_bytes / 3, - "Pi journal did not eliminate cumulative context copies: stored={} naive={naive_bytes}", - stored.len() - ); - assert!(!String::from_utf8_lossy(&stored).contains("\"messages\"")); + } - let through = stored.len() as u64; - let mut reader = JournalReader::open(&path, through).await.unwrap().unwrap(); - let mut contexts = Vec::new(); - while let Some(entry) = reader.next_entry().await.unwrap() { - contexts.push( - entry - .payload - .pointer("/event/messages") - .and_then(serde_json::Value::as_array) - .cloned() - .unwrap(), + #[tokio::test] + async fn every_source_deduplicates_repeated_payloads_and_resumes_losslessly() { + for source in crate::translate::Registry::default_agents().sources() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let mut expected = Vec::new(); + let mut messages = Vec::new(); + let mut raw_bytes = 0; + for index in 0..20 { + messages.push(serde_json::json!({"role":"user","content":format!("{index}:{}", "x".repeat(4096))})); + let mut event = pi_context(messages.clone(), index); + event.source = source.clone(); + raw_bytes += serde_json::to_vec(&event.redacted()).unwrap().len() + 1; + writer.append(&event).await.unwrap(); + expected.push(event); + } + drop(writer); + let stored = tokio::fs::read_to_string(&path).await.unwrap(); + assert!( + stored.len() < raw_bytes / 3, + "{source} should benefit from the shared payload codec" ); + assert!(stored.contains(codec::DELTA_RECORD)); + let mut reader = JournalReader::open(&path, stored.len() as u64) + .await + .unwrap() + .unwrap(); + let mut boundary = 0; + for (index, event) in expected.iter().enumerate() { + let entry = reader.next_record().await.unwrap().unwrap(); + let JournalRecord::Event(replayed) = entry.record else { + panic!("expected event") + }; + assert_eq!(replayed.payload, event.payload, "{source} event {index}"); + boundary = entry.through; + // Restore directly in the middle of a delta chain, with no + // prefix replay and without assumptions about agent schemas. + if index == 9 { + let saved = serde_json::to_vec(reader.decoder_context()).unwrap(); + let context = serde_json::from_slice(&saved).unwrap(); + reader = JournalReader::resume(&path, boundary, stored.len() as u64, context) + .await + .unwrap(); + } + } + assert_eq!(boundary, stored.len() as u64); } - assert_eq!(contexts.len(), 21); - assert_eq!(contexts[19].len(), 20); - assert_eq!(contexts[20], compacted); } #[tokio::test] - async fn pi_streaming_updates_do_not_persist_growing_partial_messages() { + async fn legacy_pi_deltas_recover_and_mix_with_complete_new_events() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("pi.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let marker = "growing-partial-marker".repeat(4096); - let event = Envelope { - source: "pi".into(), - source_version: None, - plugin_version: None, - session_id: "pi-session".into(), - event: "message_update".into(), - ts_ms: 1, - managed_run_id: None, - capture: None, - payload: serde_json::json!({ - "event": { - "type": "message_update", - "assistantMessageEvent": { - "type": "text_delta", - "partial": marker, - }, - "message": {"role": "assistant", "content": marker}, - } - }), - route: None, - config: None, - }; - writer.append(&event).await.unwrap(); - drop(writer); - - let stored = tokio::fs::read(&path).await.unwrap(); - assert!(!String::from_utf8_lossy(&stored).contains("growing-partial-marker")); - - let mut reader = JournalReader::open(&path, stored.len() as u64) + let first = pi_context( + vec![serde_json::json!({"role":"user","content":"old history"})], + 1, + ); + let mut delta = pi_context(Vec::new(), 2).redacted(); + delta.payload["event"] + .as_object_mut() + .unwrap() + .remove("messages"); + delta.payload["event"][LEGACY_PI_MESSAGES_DELTA] = serde_json::json!({"common_prefix":1,"suffix":[{"role":"assistant","content":"legacy delta"}]}); + let first_line = format!("{}\n", serde_json::to_string(&first.redacted()).unwrap()); + let old_log = format!("{}{}\n", first_line, serde_json::to_string(&delta).unwrap()); + tokio::fs::write(&path, old_log.as_bytes()).await.unwrap(); + let mut prefix = JournalReader::open(&path, first_line.len() as u64) .await .unwrap() .unwrap(); - let replayed = reader.next_entry().await.unwrap().unwrap(); - assert_eq!( - replayed - .payload - .pointer("/event/assistantMessageEvent/type"), - Some(&serde_json::json!("text_delta")) - ); - } - - #[tokio::test] - async fn pi_provider_requests_persist_only_request_configuration() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("pi.ndjson"); + prefix.next_entry().await.unwrap().unwrap(); + let decoder = prefix.decoder_context().clone(); let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let event = Envelope { - source: "pi".into(), - source_version: None, - plugin_version: None, - session_id: "pi-session".into(), - event: "before_provider_request".into(), - ts_ms: 1, - managed_run_id: None, - capture: None, - payload: serde_json::json!({ - "event": { - "type": "before_provider_request", - "payload": { - "system": "system-prompt-marker", - "messages": [{"role": "user", "content": "message-marker"}], - "tools": [{"name": "read", "input_schema": {"type": "object"}}], - "max_tokens": 1024, - }, - } - }), - route: None, - config: None, - }; - writer.append(&event).await.unwrap(); + let mut native = pi_context( + vec![serde_json::json!({"role":"user","content":"new native snapshot"})], + 3, + ); + // Even an identically named field inside a native payload must be + // preserved. Only untagged old records use the compatibility codec. + native.payload["event"][LEGACY_PI_MESSAGES_DELTA] = + serde_json::json!({"native_field":"preserve"}); + let tail = writer.append(&native).await.unwrap(); drop(writer); - - let stored = tokio::fs::read(&path).await.unwrap(); - let stored_text = String::from_utf8_lossy(&stored); - assert!(!stored_text.contains("system-prompt-marker")); - assert!(!stored_text.contains("message-marker")); - - let mut reader = JournalReader::open(&path, stored.len() as u64) + let mut reader = JournalReader::resume(&path, first_line.len() as u64, tail, decoder) .await - .unwrap() .unwrap(); - let replayed = reader.next_entry().await.unwrap().unwrap(); + let expanded = reader.next_entry().await.unwrap().unwrap(); assert_eq!( - replayed.payload.pointer("/event/payload"), - Some(&serde_json::json!({ - "max_tokens": 1024, - "tools": [{"type": "function", "function": { - "name": "read", - "parameters": {"type": "object"}, - }}], - })) + expanded.payload["event"]["messages"], + serde_json::json!([{"role":"user","content":"old history"},{"role":"assistant","content":"legacy delta"}]) ); - } - - #[test] - fn pi_journaled_request_configuration_replays_unchanged() { - let schema = serde_json::json!({"type": "object"}); - for payload in [ - serde_json::json!({ - "max_tokens": 1024, - "tools": [ - {"name": "read", "input_schema": schema, "strict": true}, - {"type": "web_search_20250305", "name": "web_search"}, - ], - "tool_choice": {"type": "tool", "name": "read", "disable_parallel_tool_use": true}, - }), - serde_json::json!({"config": { - "temperature": 0.5, - "tools": [{"functionDeclarations": [{"name": "read", "parameters": schema}]}, {"googleSearch": {}}], - "toolConfig": {"functionCallingConfig": {"mode": "ANY"}}, - }}), - serde_json::json!({ - "toolConfig": {"tools": [{"toolSpec": {"name": "read", "inputSchema": {"json": schema}}}]}, - "inferenceConfig": {"maxTokens": 512}, - }), - ] { - let journaled = - serde_json::Value::Object(crate::translate::pi_request_config(&payload)); - assert_eq!( - serde_json::Value::Object(crate::translate::pi_request_config(&journaled)), - journaled - ); - } + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + native.payload + ); + assert!(reader.decoder_context().legacy_pi.is_empty()); } #[tokio::test] diff --git a/bt-daemon/src/journal/codec.rs b/bt-daemon/src/journal/codec.rs new file mode 100644 index 00000000..f35cd27b --- /dev/null +++ b/bt-daemon/src/journal/codec.rs @@ -0,0 +1,300 @@ +//! Lossless payload encoding shared by every source journal. Repeated event +//! kinds may reference their previous payload, with a bounded per-session cache. +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{collections::VecDeque, io::Write}; + +// These bounds are part of the v1 decoding contract. +const MIN_BYTES: usize = 4 * 1024; +const MAX_BASES: usize = 8; +const MAX_BYTES: usize = 8 * 1024 * 1024; +pub(super) const DELTA_RECORD: &str = "payload_delta_v1"; + +#[derive(Clone, Default, Serialize, Deserialize)] +pub(crate) struct PayloadCache { + entries: VecDeque, +} +#[derive(Clone, Serialize, Deserialize)] +struct Base { + event: String, + through: u64, + json: String, +} +#[derive(Serialize, Deserialize)] +pub(super) struct Delta { + base: u64, + prefix: usize, + suffix: usize, + insert: String, +} + +/// Stop serialization once the cache budget is exceeded. Large events remain +/// complete raw records without allocating another unbounded payload copy. +struct LimitedJson { + bytes: Vec, + limit: usize, + exceeded: bool, +} +impl Write for LimitedJson { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > self.limit.saturating_sub(self.bytes.len()) { + self.exceeded = true; + return Err(std::io::Error::other("journal codec cache budget exceeded")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +pub(super) fn cache_json(event: &str, payload: &Value) -> anyhow::Result> { + if event.len() >= MAX_BYTES { + return Ok(None); + } + let mut output = LimitedJson { + bytes: Vec::new(), + limit: MAX_BYTES - event.len(), + exceeded: false, + }; + let result = serde_json::to_writer(&mut output, payload); + if output.exceeded { + return Ok(None); + } + result?; + if output.bytes.len() < MIN_BYTES { + return Ok(None); + } + Ok(Some(String::from_utf8(output.bytes)?)) +} + +impl PayloadCache { + pub(super) fn encode( + &self, + event: &str, + payload: &Value, + ) -> anyhow::Result<(Option, Option)> { + let Some(json) = cache_json(event, payload)? else { + return Ok((None, None)); + }; + let Some(base) = self.entries.iter().find(|base| base.event == event) else { + return Ok((Some(json), None)); + }; + let mut prefix = base + .json + .bytes() + .zip(json.bytes()) + .take_while(|(a, b)| a == b) + .count(); + while !base.json.is_char_boundary(prefix) || !json.is_char_boundary(prefix) { + prefix -= 1; + } + let remaining = (base.json.len() - prefix).min(json.len() - prefix); + let mut suffix = base + .json + .bytes() + .rev() + .zip(json.bytes().rev()) + .take(remaining) + .take_while(|(a, b)| a == b) + .count(); + while !base.json.is_char_boundary(base.json.len() - suffix) + || !json.is_char_boundary(json.len() - suffix) + { + suffix -= 1; + } + let delta = Delta { + base: base.through, + prefix, + suffix, + insert: json[prefix..json.len() - suffix].to_owned(), + }; + // Account for the extra record tag and avoid expanding small payloads. + let worthwhile = serde_json::to_vec(&delta)?.len() + 16 < json.len(); + Ok((Some(json), worthwhile.then_some(delta))) + } + + pub(super) fn record(&mut self, event: &str, through: u64, json: Option) { + self.entries.retain(|base| base.event != event); + let Some(json) = json else { + return; + }; + if json.len() > MAX_BYTES.saturating_sub(event.len()) { + return; + } + while self.entries.len() >= MAX_BASES + || self + .entries + .iter() + .map(|base| base.json.len() + base.event.len()) + .sum::() + + json.len() + + event.len() + > MAX_BYTES + { + self.entries.pop_front(); + } + self.entries.push_back(Base { + event: event.to_owned(), + through, + json, + }); + } + + pub(super) fn decode(&self, event: &str, encoded: Value) -> anyhow::Result<(Value, String)> { + let delta: Delta = serde_json::from_value(encoded)?; + let base = self + .entries + .iter() + .find(|base| base.event == event && base.through == delta.base) + .ok_or_else(|| anyhow::anyhow!("journal payload delta base is unavailable"))?; + let kept = delta + .prefix + .checked_add(delta.suffix) + .ok_or_else(|| anyhow::anyhow!("invalid journal delta bounds"))?; + anyhow::ensure!( + kept <= base.json.len() + && base.json.is_char_boundary(delta.prefix) + && base.json.is_char_boundary(base.json.len() - delta.suffix), + "invalid journal delta bounds" + ); + let budget = MAX_BYTES.saturating_sub(event.len()); + anyhow::ensure!( + delta.insert.len() <= budget && kept <= budget - delta.insert.len(), + "journal delta exceeds codec budget" + ); + let mut json = String::with_capacity(kept + delta.insert.len()); + json.push_str(&base.json[..delta.prefix]); + json.push_str(&delta.insert); + json.push_str(&base.json[base.json.len() - delta.suffix..]); + Ok((serde_json::from_str(&json)?, json)) + } + + pub(super) fn validate(&self) -> anyhow::Result<()> { + anyhow::ensure!( + self.entries.len() <= MAX_BASES + && self + .entries + .iter() + .map(|base| base.json.len() + base.event.len()) + .sum::() + <= MAX_BYTES, + "journal decoder checkpoint exceeds codec budget" + ); + Ok(()) + } +} + +#[derive(Clone, Default, Serialize)] +pub(crate) struct DecoderContext { + pub(crate) legacy_pi: Vec, + pub(super) payloads: PayloadCache, +} +impl<'de> Deserialize<'de> for DecoderContext { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Stored { + Legacy(Vec), + Current { + legacy_pi: Vec, + payloads: PayloadCache, + }, + } + let context = match Stored::deserialize(deserializer)? { + Stored::Legacy(legacy_pi) => Self { + legacy_pi, + payloads: PayloadCache::default(), + }, + Stored::Current { + legacy_pi, + payloads, + } => Self { + legacy_pi, + payloads, + }, + }; + context + .payloads + .validate() + .map_err(serde::de::Error::custom)?; + Ok(context) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn unicode_prefix_suffix_deltas_are_lossless_and_reject_missing_bases() { + let first = json!({"data":format!("{}é{}", "é".repeat(1000), "tail".repeat(1000))}); + let next = json!({"data":format!("{}ê{}", "é".repeat(1000), "tail".repeat(1000))}); + let mut cache = PayloadCache::default(); + let (base, delta) = cache.encode("arbitrary-event", &first).unwrap(); + assert!(delta.is_none()); + cache.record("arbitrary-event", 123, base); + let (_, delta) = cache.encode("arbitrary-event", &next).unwrap(); + let encoded = serde_json::to_value(delta.unwrap()).unwrap(); + assert_eq!( + cache.decode("arbitrary-event", encoded.clone()).unwrap().0, + next + ); + assert!(PayloadCache::default() + .decode("arbitrary-event", encoded.clone()) + .is_err()); + let mut invalid = encoded; + invalid["prefix"] = json!(usize::MAX); + assert!(cache.decode("arbitrary-event", invalid).is_err()); + } + + #[test] + fn cache_limits_evict_old_bases_and_preserve_oversized_payloads() { + let mut cache = PayloadCache::default(); + let payload = json!({"data":"x".repeat(MAX_BYTES / 3)}); + for index in 0..MAX_BASES + 2 { + let name = format!("event-{index}"); + let (json, _) = cache.encode(&name, &payload).unwrap(); + cache.record(&name, index as u64, json); + cache.validate().unwrap(); + } + assert!(cache.entries.iter().all(|base| base.event != "event-0")); + let (small, delta) = cache.encode("small", &json!({"value":1})).unwrap(); + assert!( + small.is_none() && delta.is_none(), + "small hook payloads should retain the direct serialization path" + ); + let oversized = json!({"data":"x".repeat(MAX_BYTES + 1)}); + let (json, delta) = cache.encode("large", &oversized).unwrap(); + assert!(json.is_none()); + assert!(delta.is_none()); + cache.record("large", 100, json); + cache.record("large", 101, Some("x".repeat(MAX_BYTES + 1))); + cache.validate().unwrap(); + assert!(cache.entries.iter().all(|base| base.event != "large")); + } + + #[test] + fn checkpoints_read_legacy_decoder_state_and_preserve_shared_bases() { + let legacy = vec![json!({"role":"user","content":"legacy"})]; + let context: DecoderContext = serde_json::from_value(json!(legacy)).unwrap(); + assert_eq!(context.legacy_pi, legacy); + let mut context = DecoderContext::default(); + let payload = json!({"messages":["content".repeat(1000)]}); + let (json, _) = context.payloads.encode("message", &payload).unwrap(); + context.payloads.record("message", 100, json); + let saved = serde_json::to_vec(&context).unwrap(); + let restored: DecoderContext = serde_json::from_slice(&saved).unwrap(); + let (_, delta) = context.payloads.encode("message", &payload).unwrap(); + assert_eq!( + restored + .payloads + .decode("message", serde_json::to_value(delta.unwrap()).unwrap()) + .unwrap() + .0, + payload + ); + } +} diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 0a826dfa..921a90fa 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -29,7 +29,7 @@ const CHECKPOINT_BYTES: u64 = 1024 * 1024; pub(crate) struct RecoveryContext { format: u32, translator: Box, - pub decoder: Vec, + pub decoder: crate::journal::DecoderContext, capture: Option, bounds: HashMap, pub templates: HashMap, @@ -151,6 +151,7 @@ impl Translation { committed, checkpoint_position: 0, checkpoint_events: 0, + checkpoint_work_bytes: 0, checkpoint_safe: true, state, publication: None, @@ -262,6 +263,7 @@ enum State { } struct Publication { input: u64, + decoded_bytes: u64, ordinal: u64, observation: bool, capture: Option, @@ -290,6 +292,7 @@ struct Producer { committed: u64, checkpoint_position: u64, checkpoint_events: u64, + checkpoint_work_bytes: u64, checkpoint_safe: bool, state: State, publication: Option, @@ -475,7 +478,7 @@ impl Producer { translator: serde_json::value::RawValue::from_string(String::from_utf8( state, )?)?, - decoder: reader.decoder_context().to_vec(), + decoder: reader.decoder_context().clone(), capture, bounds, templates, @@ -505,6 +508,7 @@ impl Producer { self.journal.save_translation_checkpoint(checkpoint).await?; self.checkpoint_position = self.committed; self.checkpoint_events = 0; + self.checkpoint_work_bytes = 0; } Ok(()) } @@ -593,6 +597,7 @@ impl Producer { self.position = 0; self.checkpoint_position = 0; self.checkpoint_events = 0; + self.checkpoint_work_bytes = 0; self.checkpoint_safe = true; if let Some(checkpoint) = self.journal.translation_checkpoint().await? { if checkpoint.version == self.options.version && checkpoint.input <= self.committed { @@ -783,6 +788,7 @@ impl Producer { }; self.publication = Some(Publication { input, + decoded_bytes: entry.decoded_bytes, ordinal: 0, observation, capture, @@ -892,6 +898,10 @@ impl Producer { self.phase = Boundary::Journal; let checkpoint = if self.checkpoint_events + 1 >= CHECKPOINT_EVENTS || pending.input.saturating_sub(self.checkpoint_position) >= CHECKPOINT_BYTES + || self + .checkpoint_work_bytes + .saturating_add(pending.decoded_bytes) + >= CHECKPOINT_BYTES { self.checkpoint_context(pending.input).await? } else { @@ -904,13 +914,25 @@ impl Producer { if checkpointed { self.checkpoint_position = pending.input; self.checkpoint_events = 0; + self.checkpoint_work_bytes = 0; } else { self.checkpoint_events += 1; + self.checkpoint_work_bytes = self + .checkpoint_work_bytes + .saturating_add(pending.decoded_bytes); } self.committed = pending.input; self.progress .committed .store(pending.input, Ordering::Relaxed); + } else { + // Restoring a checkpoint replays the committed suffix too. + // Count that work so repeated short-lived processes cannot + // keep extending the distance from the same old checkpoint. + self.checkpoint_events += 1; + self.checkpoint_work_bytes = self + .checkpoint_work_bytes + .saturating_add(pending.decoded_bytes); } self.position = pending.input; Ok(()) @@ -1052,6 +1074,7 @@ mod tests { committed: 0, checkpoint_position: 0, checkpoint_events: 0, + checkpoint_work_bytes: 0, checkpoint_safe: true, state: State::Processing, publication: None, @@ -1071,6 +1094,89 @@ mod tests { assert!(checkpoint.await.unwrap().unwrap().is_some()); } + #[tokio::test] + async fn checkpoint_budget_counts_decoded_payloads_after_compression() { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let source = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) + .await + .unwrap(); + let env = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{"history":"x".repeat(256 * 1024)}})).unwrap(); + let mut first = 0; + let mut tail = 0; + for index in 0..10 { + tail = writer.lock().await.append(&env).await.unwrap(); + if index == 0 { + first = tail; + } + } + assert!( + tail < CHECKPOINT_BYTES, + "fixture must compress below the physical byte threshold" + ); + source.advance(tail).await.unwrap(); + source.barrier().await.unwrap(); + let checkpoint = source + .journal + .translation_checkpoint() + .await + .unwrap() + .expect("decoded work must trigger a checkpoint before 64 inputs"); + assert!(checkpoint.input > first); + source.shutdown().await; + } + + #[tokio::test] + async fn checkpoint_window_accounts_for_committed_suffix_on_restart() { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let source = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) + .await + .unwrap(); + let env = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{}})).unwrap(); + let mut tail = 0; + for _ in 0..2 * CHECKPOINT_EVENTS - 1 { + tail = writer.lock().await.append(&env).await.unwrap(); + } + source.advance(tail).await.unwrap(); + source.barrier().await.unwrap(); + let before = source + .journal + .translation_checkpoint() + .await + .unwrap() + .unwrap() + .input; + assert!(before < tail); + let task = source.task.lock().await.take().unwrap(); + task.abort(); + let _ = task.await; + tail = writer.lock().await.append(&env).await.unwrap(); + let resumed = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) + .await + .unwrap(); + resumed.advance(tail).await.unwrap(); + resumed.barrier().await.unwrap(); + assert_eq!( + resumed + .journal + .translation_checkpoint() + .await + .unwrap() + .unwrap() + .input, + tail, + "the 63 reconstructed inputs must count toward the next periodic checkpoint" + ); + resumed.shutdown().await; + } + #[tokio::test] async fn every_agent_journal_resumes_at_its_persisted_byte_offset() { for source in Registry::default_agents().sources() { @@ -1134,8 +1240,9 @@ mod tests { } #[tokio::test] - async fn checkpoint_seeks_past_completed_history_and_restores_pi_delta_context() { - for source in ["debug", "pi"] { + async fn checkpoint_seeks_past_history_for_native_and_legacy_journals() { + for format in ["debug", "pi", "legacy-pi"] { + let source = if format == "legacy-pi" { "pi" } else { format }; let tmp = tempfile::tempdir().unwrap(); let path = crate::journal::source_journal_path(tmp.path(), source, "checkpoint"); let writer = Arc::new(tokio::sync::Mutex::new( @@ -1159,10 +1266,32 @@ mod tests { plugin_version: None, }; let mut boundary = 0; + let mut legacy = String::new(); for i in 0..CHECKPOINT_EVENTS + 3 { env.ts_ms = i as i64; - env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); - boundary = writer.lock().await.append(&env).await.unwrap(); + env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared".repeat(1024)},{"role":"assistant","content":i}]}}); + if format == "legacy-pi" { + let mut event = env.redacted(); + event.payload["event"] + .as_object_mut() + .unwrap() + .remove("messages"); + let suffix = if i == 0 { + env.payload["event"]["messages"].clone() + } else { + serde_json::json!([{"role":"assistant","content":i}]) + }; + event.payload["event"]["_bt_messages_delta"] = serde_json::json!({"common_prefix": if i == 0 { 0 } else { 1 }, "suffix":suffix}); + legacy.push_str(&serde_json::to_string(&event).unwrap()); + legacy.push('\n'); + boundary = legacy.len() as u64; + } else { + boundary = writer.lock().await.append(&env).await.unwrap(); + } + } + if format == "legacy-pi" { + tokio::fs::write(&path, legacy).await.unwrap(); + *writer.lock().await = JournalWriter::open_path(&path).await.unwrap(); } translation.advance(boundary).await.unwrap(); translation.barrier().await.unwrap(); @@ -1176,6 +1305,12 @@ mod tests { checkpoint.input < boundary, "test must replay a suffix after a periodic checkpoint" ); + let saved = RecoveryContext::decode(&checkpoint, "test", boundary).unwrap(); + assert_eq!( + saved.decoder.legacy_pi.is_empty(), + format != "legacy-pi", + "only legacy delta records should need a saved journal decoder base" + ); let task = translation.task.lock().await.take().unwrap(); task.abort(); let _ = task.await; // crash, without the idle checkpoint @@ -1508,11 +1643,13 @@ mod tests { committed: 0, checkpoint_position: 0, checkpoint_events: 0, + checkpoint_work_bytes: 0, checkpoint_safe: true, state: State::Processing, phase: Boundary::Journal, publication: Some(Publication { input: 10, + decoded_bytes: 0, ordinal: 0, observation: true, capture: None, diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 04d6a1a6..82e4f634 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -337,7 +337,7 @@ pub(super) async fn recover_ingress_overflow(daemon: &Arc) { checkpoints = reader; start = position; } - let decoder = checkpoints.decoder_context().to_vec(); + let decoder = checkpoints.decoder_context().clone(); let mut acknowledged_by_route: HashMap = HashMap::new(); while let Ok(Some(entry)) = checkpoints.next_record().await { if let journal::JournalRecord::DeliveryCheckpoint { route, through } = entry.record { diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 0ddae534..d5aae34d 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -819,7 +819,7 @@ mod tests { } #[tokio::test] - async fn startup_overflow_keeps_pi_delta_context_across_recovery_advances() { + async fn startup_overflow_keeps_pi_context_across_recovery_advances() { let tmp = tempfile::tempdir().unwrap(); let (ready, startup) = oneshot::channel(); let daemon = Daemon::new_with_startup( diff --git a/bt-daemon/src/translate/mod.rs b/bt-daemon/src/translate/mod.rs index 78c29ada..90c707db 100644 --- a/bt-daemon/src/translate/mod.rs +++ b/bt-daemon/src/translate/mod.rs @@ -37,7 +37,6 @@ mod muse; mod opencode; mod pi; pub(crate) use claude::successful_subagent_handback_message; -pub(crate) use pi::request_config as pi_request_config; mod origins; mod recent; pub(crate) use origins::WeakOrigins; diff --git a/bt-daemon/src/translate/pi.rs b/bt-daemon/src/translate/pi.rs index 93e90746..6ea26e26 100644 --- a/bt-daemon/src/translate/pi.rs +++ b/bt-daemon/src/translate/pi.rs @@ -1414,3 +1414,37 @@ fn bash_status(tool: &str, v: Option<&Value>) -> Option { .any(|prefix| status.starts_with(prefix)) .then(|| status.to_string()) } + +#[cfg(test)] +mod legacy_journal_tests { + use super::*; + #[test] + fn legacy_request_config_normalization_is_idempotent() { + let schema = serde_json::json!({"type": "object"}); + for payload in [ + serde_json::json!({ + "max_tokens": 1024, + "tools": [ + {"name": "read", "input_schema": schema, "strict": true}, + {"type": "web_search_20250305", "name": "web_search"}, + ], + "tool_choice": {"type": "tool", "name": "read", "disable_parallel_tool_use": true}, + }), + serde_json::json!({"config": { + "temperature": 0.5, + "tools": [{"functionDeclarations": [{"name": "read", "parameters": schema}]}, {"googleSearch": {}}], + "toolConfig": {"functionCallingConfig": {"mode": "ANY"}}, + }}), + serde_json::json!({ + "toolConfig": {"tools": [{"toolSpec": {"name": "read", "inputSchema": {"json": schema}}}]}, + "inferenceConfig": {"maxTokens": 512}, + }), + ] { + let journaled = serde_json::Value::Object(request_config(&payload)); + assert_eq!( + serde_json::Value::Object(request_config(&journaled)), + journaled + ); + } + } +} diff --git a/bt-daemon/tests/pipeline.rs b/bt-daemon/tests/pipeline.rs index 471a49ad..0590902d 100644 --- a/bt-daemon/tests/pipeline.rs +++ b/bt-daemon/tests/pipeline.rs @@ -1257,7 +1257,12 @@ async fn events_are_ordered_journaled_and_emitted() { let events: Vec = jlines .iter() - .filter(|line| !line.contains("\"_bt_record_type\"")) + .filter(|line| { + serde_json::from_str::(line) + .unwrap() + .get("event") + .is_some() + }) .map(|l| { serde_json::from_str::(l).unwrap()["event"] .as_str() @@ -1309,7 +1314,10 @@ async fn session_ids_longer_than_a_file_name_are_journaled_and_emitted() { assert_eq!( jtext .lines() - .filter(|line| !line.contains("\"_bt_record_type\"")) + .filter(|line| serde_json::from_str::(line) + .unwrap() + .get("event") + .is_some()) .count(), 2 ); @@ -1976,7 +1984,10 @@ async fn cold_worker_rebuilds_acknowledged_journal_without_redelivery() { assert_eq!( journal .lines() - .filter(|line| !line.contains("\"_bt_record_type\"")) + .filter(|line| serde_json::from_str::(line) + .unwrap() + .get("event") + .is_some()) .count(), 2, "the event history remains separate from ordered pipeline controls" @@ -3490,7 +3501,7 @@ async fn packaged_grok_hook_replays_bounded_transcripts_to_isolated_debug_routes let journal: Vec = journal_text .lines() .map(|line| serde_json::from_str(line).unwrap()) - .filter(|record: &serde_json::Value| record.get("_bt_record_type").is_none()) + .filter(|record: &serde_json::Value| record.get("event").is_some()) .collect(); assert_eq!(journal.len(), 2); assert_eq!(journal[0]["source"], "grok"); From 1f513b50c118f5d49178ea853b6e8caa0be4cd6d Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 20:54:44 +0800 Subject: [PATCH 08/19] Wait for SQLite tracking writers before reading transaction state --- bt-daemon/src/pipeline/journal.rs | 44 ++++++++++++++++++++++---- bt-daemon/tests/distributed_tracing.rs | 30 ++++++++++++------ 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/bt-daemon/src/pipeline/journal.rs b/bt-daemon/src/pipeline/journal.rs index 4fa91e22..da3dbd1c 100644 --- a/bt-daemon/src/pipeline/journal.rs +++ b/bt-daemon/src/pipeline/journal.rs @@ -1,4 +1,8 @@ //! Concrete durable channel: one ordered SpanOp log and a head per route. +//! +//! Tracking mutations acquire SQLite write access before reading. Deferred +//! read-to-write upgrades can fail without waiting when another connection +//! writes concurrently, even with a busy timeout. use super::live::{Multiplexer, Receiver, Record, SharedOp, BYTE_BUDGET, WINDOW}; use super::{ delivery::InputPort, @@ -331,7 +335,7 @@ impl SpanJournal { checkpoint: TranslationCheckpoint, ) -> anyhow::Result<()> { self.with_db(move |db| { - let tx = db.transaction()?; + let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let head: u64 = tx.query_row( "SELECT coalesce(max(input),0) FROM batches WHERE committed=1", [], @@ -354,7 +358,7 @@ impl SpanJournal { /// Restart abandons only invisible receipts; a committed identity is never rewritten. pub async fn begin(&self, input: u64, observation: bool) -> anyhow::Result { self.with_db(move |db| { - let tx = db.transaction()?; + let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let done: Option = tx .query_row( "SELECT committed FROM batches WHERE input=?", @@ -530,7 +534,7 @@ impl SpanJournal { .map(StoredOp::encode) .collect::>>()?; self.with_db(move |db| { - let tx = db.transaction()?; + let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let done: bool = tx.query_row( "SELECT committed FROM batches WHERE input=?", [input], @@ -614,7 +618,7 @@ impl SpanJournal { ) -> anyhow::Result { let tail = self .with_db(move |db| { - let tx = db.transaction()?; + let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let previous: u64 = tx.query_row( "SELECT coalesce(max(input),0) FROM batches WHERE committed=1", [], @@ -678,7 +682,7 @@ impl SpanJournal { let key = route.clone(); let position = self .with_db(move |db| { - let tx = db.transaction()?; + let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; // Auth can register a named-profile obligation before learning // its stable ID. Move that unresolved placeholder to the resolved // identity instead of leaving a second head that can never run. @@ -895,7 +899,7 @@ impl SpanJournal { ) -> anyhow::Result> { let route = route.clone(); self.with_db(move |db| { - let tx = db.transaction()?; + let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let updates = { let mut query = tx.prepare( "SELECT route,failure,settings FROM heads WHERE failure IS NOT NULL AND settings_through anyhow::Result> { - let record = Arc::new(RecordedSession { - source: source.into(), - ..RecordedSession::default() - }); - self.sessions + // Reopening a delivery sink must preserve events already accepted before + // a daemon restart, just as the destination does. + let record = self + .sessions .lock() .unwrap() - .insert(session_id.into(), record.clone()); + .entry(session_id.into()) + .or_insert_with(|| { + Arc::new(RecordedSession { + source: source.into(), + ..RecordedSession::default() + }) + }) + .clone(); Ok(Box::new(RecordingSink { record })) } } @@ -3307,10 +3313,14 @@ async fn flush(session: &str, socket: &Path) { let result = flush_session(session, socket, 5_000) .await .unwrap_or_else(|error| panic!("flush {session}: {error}")); - assert!( - result.flushed && result.pending == 0, - "session {session} did not fully flush: {result:?}" - ); + if !result.flushed || result.pending != 0 { + let status = run_status(StatusArgs { + socket: Some(socket.to_owned()), + session_id: Some(session.to_owned()), + }) + .await; + panic!("session {session} did not fully flush: {result:?}; status: {status:?}"); + } } async fn start_daemon() -> ( From d4db224dcf5c1cad98eb7661066b0fad34f96e1a Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 21:03:18 +0800 Subject: [PATCH 09/19] Keep recovery fixtures faithful to native event and delivery behavior --- bt-daemon/tests/distributed_tracing.rs | 24 ++++++++++++++++++++++-- bt-daemon/tests/root_provenance.rs | 15 +++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/bt-daemon/tests/distributed_tracing.rs b/bt-daemon/tests/distributed_tracing.rs index 0455dcec..e06ca1d2 100644 --- a/bt-daemon/tests/distributed_tracing.rs +++ b/bt-daemon/tests/distributed_tracing.rs @@ -42,6 +42,7 @@ struct RecordingSinkFactory { } impl RecordingSinkFactory { + #[track_caller] fn session(&self, session_id: &str) -> Arc { self.sessions .lock() @@ -3031,13 +3032,28 @@ async fn pending_restart_preserves_current_candidates_and_decided_legacy_links() if format != "current" { let again_recording = Arc::new(RecordingSinkFactory::default()); let again = spawn_daemon(socket.clone(), data_dir, again_recording.clone()).await; + // Exercise new work after restart. A Stop for a turn already + // completed during recovery may correctly produce no output and + // therefore never create a delivery sink. + forward_all( + &mut fixtures.start_turn( + AgentKind::Claude, + "bounded-child", + &ProcessTree::child(40_002, 40_001, &ProcessTree::root(40_000)), + "new work after upgrade", + 1_702_100_000_050, + ), + &socket, + &host, + ) + .await; forward( fixtures.close_session( AgentKind::Claude, "bounded-child", &ProcessTree::child(40_002, 40_001, &ProcessTree::root(40_000)), "after upgrade", - 1_702_100_000_050, + 1_702_100_000_060, ), &socket, &host, @@ -3045,6 +3061,10 @@ async fn pending_restart_preserves_current_candidates_and_decided_legacy_links() .await; flush("bounded-child", &socket).await; let child = again_recording.session("bounded-child"); + assert!(inserted_rows(&child).iter().any(|row| row + .input + .as_ref() + .is_some_and(|input| input.to_string().contains("new work after upgrade")))); assert!(child .configs .lock() @@ -3313,7 +3333,7 @@ async fn flush(session: &str, socket: &Path) { let result = flush_session(session, socket, 5_000) .await .unwrap_or_else(|error| panic!("flush {session}: {error}")); - if !result.flushed || result.pending != 0 { + if !result.flushed || result.pending != 0 || result.accepted_sessions == 0 { let status = run_status(StatusArgs { socket: Some(socket.to_owned()), session_id: Some(session.to_owned()), diff --git a/bt-daemon/tests/root_provenance.rs b/bt-daemon/tests/root_provenance.rs index 3613c675..5ed19b84 100644 --- a/bt-daemon/tests/root_provenance.rs +++ b/bt-daemon/tests/root_provenance.rs @@ -493,8 +493,19 @@ async fn legacy_acknowledged_spans_receive_late_updates_without_inventing_origin let raw = std::fs::read_to_string(&journal).unwrap(); let mut history = raw .lines() - .filter(|line| !line.contains("_bt_record_type")) - .map(|line| format!("{line}\n")) + .filter_map(|line| { + let mut record: Value = serde_json::from_str(line).unwrap(); + record.get("event")?; + // These small fixture payloads are full records. Remove the new + // framing to model the untagged legacy format faithfully. + assert_eq!(record["_bt_record_type"], "event"); + record.as_object_mut().unwrap().remove("_bt_record_type"); + record + .as_object_mut() + .unwrap() + .remove("_bt_capture_sequence"); + Some(format!("{record}\n")) + }) .collect::(); let through = history.len() as u64; history.push_str(&format!("{}\n",json!({"_bt_record_type":"delivery_checkpoint", "route":capture("UserPromptSubmit","v1","native1",1_000).route, "through":through}))); From 68eb0845c434a9911e9d2e57a263a61c7f1bedfa Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 21:51:44 +0800 Subject: [PATCH 10/19] Translate captured events directly from the bounded live feed --- bt-daemon/src/journal.rs | 195 +++++++++++++++++++++++++- bt-daemon/src/journal/codec.rs | 18 +-- bt-daemon/src/journal/live.rs | 70 +++++++++ bt-daemon/src/pipeline/translation.rs | 93 ++++++++++-- bt-daemon/src/server/ingress.rs | 2 +- 5 files changed, 355 insertions(+), 23 deletions(-) create mode 100644 bt-daemon/src/journal/live.rs diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index 9207cd69..2fd35d6b 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -11,7 +11,9 @@ //! daemon that accepted them has restarted or idle-exited. mod codec; +mod live; pub(crate) use codec::DecoderContext; +pub(crate) use live::LiveRecords; use crate::wire::{BackendAuth, Envelope, RedactedEnvelope, SessionRoute}; use serde::{Deserialize, Serialize}; @@ -225,6 +227,7 @@ pub struct JournalWriter { file: tokio::fs::File, position: u64, payloads: codec::PayloadCache, + live: LiveRecords, } impl JournalWriter { @@ -242,6 +245,7 @@ impl JournalWriter { file, position, payloads: codec::PayloadCache::default(), + live: LiveRecords::default(), }) } @@ -249,6 +253,10 @@ impl JournalWriter { self.position } + pub(crate) fn live_records(&self) -> LiveRecords { + self.live.clone() + } + /// Append one redacted event and confirm its durable write before capture acknowledges it. /// /// The journal is a durability record and is never capped or truncated: @@ -276,23 +284,39 @@ impl JournalWriter { } let mut redacted = env.redacted(); let (json, delta) = self.payloads.encode(&env.event, &redacted.payload)?; - let record_type = if let Some(delta) = delta { - redacted.payload = serde_json::to_value(delta)?; - codec::DELTA_RECORD + let (record_type, original) = if let Some(delta) = delta { + let original = std::mem::replace(&mut redacted.payload, serde_json::to_value(delta)?); + (codec::DELTA_RECORD, Some(original)) } else { - "event" + ("event", None) }; let mut line = serde_json::to_vec(&Captured { envelope: &redacted, _bt_record_type: record_type, _bt_capture_sequence: sequence, })?; + if let Some(original) = original { + redacted.payload = original; + } line.push(b'\n'); self.file.write_all(&line).await?; self.file.flush().await?; self.file.sync_data().await?; + let before = self.position; self.position += line.len() as u64; - self.payloads.record(&env.event, self.position, json); + self.payloads + .record(&env.event, self.position, json.clone()); + self.live.push(live::LiveRecord { + before, + entry: JournalRecordEntry { + record: JournalRecord::Event(redacted), + through: self.position, + decoded_bytes: (line.len() as u64) + .max(json.as_ref().map_or(0, |json| json.len() as u64)), + capture_sequence: sequence, + }, + json, + }); Ok(self.position) } @@ -307,7 +331,18 @@ impl JournalWriter { self.file.write_all(&line).await?; self.file.flush().await?; self.file.sync_data().await?; + let before = self.position; self.position += line.len() as u64; + self.live.push(live::LiveRecord { + before, + entry: JournalRecordEntry { + record: JournalRecord::Control(control.clone()), + through: self.position, + decoded_bytes: line.len() as u64, + capture_sequence: None, + }, + json: None, + }); Ok(self.position) } } @@ -365,6 +400,10 @@ pub struct JournalReader { line_no: usize, position: u64, decoder: DecoderContext, + through: u64, + needs_seek: bool, + #[cfg(test)] + disk_records: u64, } impl JournalReader { @@ -372,6 +411,25 @@ impl JournalReader { &self.decoder } + /// Consume a complete captured record without touching the journal file. + /// Keep its decoder base solely for checkpoints and later disk recovery. + pub(crate) fn next_live( + &mut self, + live: &LiveRecords, + through: u64, + ) -> Option { + let record = live.take(self.position, through)?; + if let JournalRecord::Event(event) = &record.entry.record { + self.decoder.legacy_pi.clear(); + self.decoder + .payloads + .record(&event.event, record.entry.through, record.json); + } + self.position = record.entry.through; + self.needs_seek = true; + Some(record.entry) + } + /// Resume any source journal at a committed byte offset with its saved /// lossless payload cache (and, for old Pi logs, their legacy message base). pub(crate) async fn resume( @@ -399,6 +457,10 @@ impl JournalReader { line_no: 0, position, decoder: decoder_context, + through, + needs_seek: false, + #[cfg(test)] + disk_records: 0, }) } /// Open a journal for streaming, reading at most `through` bytes. @@ -420,6 +482,10 @@ impl JournalReader { line_no: 0, position: 0, decoder: DecoderContext::default(), + through, + needs_seek: false, + #[cfg(test)] + disk_records: 0, })) } @@ -445,12 +511,34 @@ impl JournalReader { /// The next journal record, including delivery checkpoints written by a /// previous daemon generation. pub async fn next_record(&mut self) -> anyhow::Result> { + if self.position >= self.through { + return Ok(None); + } + if self.needs_seek { + // Live records can skip beyond buffered disk data. Reposition only + // when an evicted or historical record actually needs recovery. + let buffered = self.reader.buffer().len(); + self.reader.consume(buffered); + self.reader + .get_mut() + .get_mut() + .seek(std::io::SeekFrom::Start(self.position)) + .await?; + self.reader + .get_mut() + .set_limit(self.through - self.position); + self.needs_seek = false; + } loop { let mut bytes = Vec::new(); let read = self.reader.read_until(b'\n', &mut bytes).await?; if read == 0 { return Ok(None); } + #[cfg(test)] + { + self.disk_records += 1; + } self.position += read as u64; self.line_no += 1; let line = std::str::from_utf8(&bytes) @@ -538,6 +626,10 @@ impl JournalReader { } pub(crate) fn extend(&mut self, through: u64) { + self.through = through; + if self.needs_seek { + return; + } let buffered = self.reader.buffer().len() as u64; self.reader .get_mut() @@ -858,6 +950,99 @@ mod tests { } } + #[tokio::test] + async fn complete_live_records_bypass_disk_and_delta_expansion_for_every_source() { + for source in crate::translate::Registry::default_agents().sources() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let live = writer.live_records(); + let mut expected = Vec::new(); + for index in 0..4 { + let mut event = pi_context(vec![], index); + event.source = source.clone(); + event.payload = + serde_json::json!({"history":"shared history".repeat(1024),"index":index}); + writer.append(&event).await.unwrap(); + expected.push(event); + } + let contents = tokio::fs::read_to_string(&path).await.unwrap(); + assert!(contents.contains(codec::DELTA_RECORD)); + let through = writer.position(); + let mut reader = JournalReader::open(&path, through).await.unwrap().unwrap(); + for expected in expected { + let entry = reader + .next_live(&live, through) + .expect("retained live event"); + let JournalRecord::Event(actual) = entry.record else { + panic!("expected event") + }; + assert_eq!(actual.payload, expected.payload, "{source}"); + } + assert!(reader.next_record().await.unwrap().is_none()); + assert_eq!(reader.disk_records, 0, "{source} read back live events"); + } + } + + #[tokio::test] + async fn evicted_records_recover_from_disk_then_rejoin_the_complete_live_feed() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let live = writer.live_records(); + let mut reader = JournalReader::open(&path, 0).await.unwrap().unwrap(); + for batch in 0..2 { + for index in 0..80 { + let mut event = pi_context(vec![], index); + event.payload = + serde_json::json!({"history":"repeated".repeat(1024),"index":batch*80+index}); + writer.append(&event).await.unwrap(); + } + let through = writer.position(); + reader.extend(through); + for index in 0..80 { + let entry = match reader.next_live(&live, through) { + Some(entry) => entry, + None => reader.next_record().await.unwrap().unwrap(), + }; + let JournalRecord::Event(event) = entry.record else { + panic!("expected event") + }; + assert_eq!(event.payload["index"], batch * 80 + index); + assert_eq!(event.payload["history"], "repeated".repeat(1024)); + } + assert!(reader.next_record().await.unwrap().is_none()); + assert_eq!(reader.disk_records, ((batch + 1) * 16) as u64); + } + } + + #[tokio::test] + async fn live_feed_bounds_decoded_bytes_and_allows_one_oversized_event() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let live = writer.live_records(); + let mut event = pi_context(vec![], 0); + event.payload = serde_json::json!({"history":"x".repeat(9 * 1024 * 1024)}); + let first = writer.append(&event).await.unwrap(); + let mut reader = JournalReader::open(&path, 0).await.unwrap().unwrap(); + assert!( + reader.next_live(&live, first).is_some(), + "an oversized event must still flow directly" + ); + let second = writer.append(&event).await.unwrap(); + event.payload = serde_json::json!({"history":"x".repeat(3 * 1024 * 1024)}); + let third = writer.append(&event).await.unwrap(); + assert!( + reader.next_live(&live, third).is_none(), + "byte pressure must evict the oversized event" + ); + reader.extend(second); + assert_eq!(reader.next_record().await.unwrap().unwrap().through, second); + assert_eq!(reader.next_live(&live, third).unwrap().through, third); + assert_eq!(reader.disk_records, 1); + } + #[tokio::test] async fn legacy_pi_deltas_recover_and_mix_with_complete_new_events() { let temp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/journal/codec.rs b/bt-daemon/src/journal/codec.rs index f35cd27b..40554eef 100644 --- a/bt-daemon/src/journal/codec.rs +++ b/bt-daemon/src/journal/codec.rs @@ -2,7 +2,7 @@ //! kinds may reference their previous payload, with a bounded per-session cache. use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::{collections::VecDeque, io::Write}; +use std::{collections::VecDeque, io::Write, sync::Arc}; // These bounds are part of the v1 decoding contract. const MIN_BYTES: usize = 4 * 1024; @@ -18,7 +18,7 @@ pub(crate) struct PayloadCache { struct Base { event: String, through: u64, - json: String, + json: Arc, } #[derive(Serialize, Deserialize)] pub(super) struct Delta { @@ -49,7 +49,7 @@ impl Write for LimitedJson { } } -pub(super) fn cache_json(event: &str, payload: &Value) -> anyhow::Result> { +pub(super) fn cache_json(event: &str, payload: &Value) -> anyhow::Result>> { if event.len() >= MAX_BYTES { return Ok(None); } @@ -66,7 +66,7 @@ pub(super) fn cache_json(event: &str, payload: &Value) -> anyhow::Result anyhow::Result<(Option, Option)> { + ) -> anyhow::Result<(Option>, Option)> { let Some(json) = cache_json(event, payload)? else { return Ok((None, None)); }; @@ -115,7 +115,7 @@ impl PayloadCache { Ok((Some(json), worthwhile.then_some(delta))) } - pub(super) fn record(&mut self, event: &str, through: u64, json: Option) { + pub(super) fn record(&mut self, event: &str, through: u64, json: Option>) { self.entries.retain(|base| base.event != event); let Some(json) = json else { return; @@ -142,7 +142,7 @@ impl PayloadCache { }); } - pub(super) fn decode(&self, event: &str, encoded: Value) -> anyhow::Result<(Value, String)> { + pub(super) fn decode(&self, event: &str, encoded: Value) -> anyhow::Result<(Value, Arc)> { let delta: Delta = serde_json::from_value(encoded)?; let base = self .entries @@ -168,7 +168,7 @@ impl PayloadCache { json.push_str(&base.json[..delta.prefix]); json.push_str(&delta.insert); json.push_str(&base.json[base.json.len() - delta.suffix..]); - Ok((serde_json::from_str(&json)?, json)) + Ok((serde_json::from_str(&json)?, json.into())) } pub(super) fn validate(&self) -> anyhow::Result<()> { @@ -271,7 +271,7 @@ mod tests { assert!(json.is_none()); assert!(delta.is_none()); cache.record("large", 100, json); - cache.record("large", 101, Some("x".repeat(MAX_BYTES + 1))); + cache.record("large", 101, Some("x".repeat(MAX_BYTES + 1).into())); cache.validate().unwrap(); assert!(cache.entries.iter().all(|base| base.event != "large")); } diff --git a/bt-daemon/src/journal/live.rs b/bt-daemon/src/journal/live.rs new file mode 100644 index 00000000..45bf3300 --- /dev/null +++ b/bt-daemon/src/journal/live.rs @@ -0,0 +1,70 @@ +//! Complete durable records awaiting their single source translator. Eviction +//! transfers the recovery obligation to the journal without delaying capture. +use super::JournalRecordEntry; +use parking_lot::Mutex; +use std::{collections::VecDeque, sync::Arc}; + +const RECORD_BUDGET: usize = 64; +const BYTE_BUDGET: u64 = 8 * 1024 * 1024; + +pub(super) struct LiveRecord { + pub before: u64, + pub entry: JournalRecordEntry, + pub json: Option>, +} + +impl LiveRecord { + fn bytes(&self) -> u64 { + self.entry + .decoded_bytes + .saturating_add(self.json.as_ref().map_or(0, |json| json.len() as u64)) + } +} + +#[derive(Default)] +struct Buffer { + records: VecDeque, + bytes: u64, +} + +#[derive(Clone, Default)] +pub(crate) struct LiveRecords(Arc>); + +impl LiveRecords { + pub(super) fn push(&self, record: LiveRecord) { + let mut buffer = self.0.lock(); + let bytes = record.bytes(); + // An oversized event travels alone, just as it does on the SpanOp feed. + while !buffer.records.is_empty() + && (buffer.records.len() >= RECORD_BUDGET + || buffer.bytes.saturating_add(bytes) > BYTE_BUDGET) + { + if let Some(evicted) = buffer.records.pop_front() { + buffer.bytes -= evicted.bytes(); + } + } + buffer.bytes = buffer.bytes.saturating_add(bytes); + buffer.records.push_back(record); + } + + pub(super) fn take(&self, position: u64, through: u64) -> Option { + let mut buffer = self.0.lock(); + // Recovery may already have consumed records whose live copies remained. + while buffer + .records + .front() + .is_some_and(|record| record.entry.through <= position) + { + if let Some(consumed) = buffer.records.pop_front() { + buffer.bytes -= consumed.bytes(); + } + } + let first = buffer.records.front()?; + if first.before != position || first.entry.through > through { + return None; + } + let record = buffer.records.pop_front()?; + buffer.bytes -= record.bytes(); + Some(record) + } +} diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 921a90fa..6b792330 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -7,7 +7,9 @@ use super::{ Stage, }; use crate::{ - journal::{ControlKind, JournalReader, JournalRecord, JournalWriter, PipelineControl}, + journal::{ + ControlKind, JournalReader, JournalRecord, JournalWriter, LiveRecords, PipelineControl, + }, translate::{AgentTranslator, OriginSnapshot, Registry, SessionCtx, SpanOp, WeakOrigins}, wire::{SessionConfig, SessionRoute}, }; @@ -118,6 +120,7 @@ impl Translation { crate::ids::session_storage_id(&options.source, &options.session_id) )); let journal = SpanJournal::open(&path).await?; + let live = options.events.lock().await.live_records(); let committed = journal.translated_through().await?; let progress = Arc::new(Progress { requested: AtomicU64::new(committed), @@ -140,6 +143,7 @@ impl Translation { recovery: recovery.clone(), translator: None, reader: None, + live, observers: Vec::new(), origins: WeakOrigins::default(), roots: Roots::default(), @@ -281,6 +285,7 @@ struct Producer { recovery: Arc>>, translator: Option>, reader: Option, + live: LiveRecords, observers: Vec, origins: WeakOrigins, roots: Roots, @@ -667,13 +672,20 @@ impl Producer { .ok_or_else(|| anyhow::anyhow!("event journal is missing"))?; reader.extend(through); self.phase = Boundary::Journal; - while let Some(entry) = self - .reader - .as_mut() - .ok_or_else(|| anyhow::anyhow!("event journal is missing"))? - .next_record() - .await? - { + loop { + let reader = self + .reader + .as_mut() + .ok_or_else(|| anyhow::anyhow!("event journal is missing"))?; + // Captured records stay complete in memory. Only an evicted or + // historical gap needs a journal read to reconstruct source state. + let entry = match reader.next_live(&self.live, through) { + Some(entry) => Some(entry), + None => reader.next_record().await?, + }; + let Some(entry) = entry else { + break; + }; let input = entry.through; let observation = !matches!(&entry.record,JournalRecord::Control(control) if matches!(control.kind,ControlKind::Retire)); let ctx = SessionCtx { @@ -1021,6 +1033,69 @@ mod tests { } } + #[tokio::test] + async fn live_translation_and_controls_do_not_read_events_back_from_disk() { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let translation = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) + .await + .unwrap(); + let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ + "source":"debug", "session_id":"checkpoint", "event":"event", "ts_ms":1, + "payload":{"history":"shared history".repeat(1024),"index":0} + })) + .unwrap(); + writer.lock().await.append(&event).await.unwrap(); + event.payload["index"] = 1.into(); + let through = writer.lock().await.append(&event).await.unwrap(); + let original = tokio::fs::read(&path).await.unwrap(); + assert!(String::from_utf8_lossy(&original).contains("payload_delta_v1")); + // A journal read would fail. The complete captured records are still + // available, so normal translation and flush controls must succeed. + let mut poisoned = original.clone(); + for byte in &mut poisoned { + if *byte != b'\n' { + *byte = b'x'; + } + } + tokio::fs::write(&path, poisoned).await.unwrap(); + translation.advance(through).await.unwrap(); + translation.barrier().await.unwrap(); + translation.control(ControlKind::Checkpoint).await.unwrap(); + let rows = translation.journal.read_after(0, 32).await.unwrap(); + let inputs = rows + .iter() + .filter_map(|(_, op)| { + let (SpanOp::Insert(row) | SpanOp::Merge(row)) = op; + row.input.as_ref() + }) + .collect::>(); + assert_eq!(inputs.len(), 2); + assert_eq!(inputs[1], &event.payload); + translation.shutdown().await; + + // A cold recovery must restore the decoder bases learned from live + // records, then expand a delta written after the saved checkpoint. + event.payload["index"] = 2.into(); + let tail = writer.lock().await.append(&event).await.unwrap(); + *writer.lock().await = JournalWriter::open_path(&path).await.unwrap(); + let resumed = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) + .await + .unwrap(); + resumed.advance(tail).await.unwrap(); + resumed.barrier().await.unwrap(); + assert_eq!(resumed.journal.translated_through().await.unwrap(), tail); + let rows = resumed.journal.read_after(0, 32).await.unwrap(); + assert!(rows.iter().any(|(_, op)| { + let (SpanOp::Insert(row) | SpanOp::Merge(row)) = op; + row.input.as_ref() == Some(&event.payload) + })); + resumed.shutdown().await; + } + struct BlockedCheckpoint { entered: Arc, release: std::sync::mpsc::Receiver<()>, @@ -1063,6 +1138,7 @@ mod tests { release: resume, })), reader: JournalReader::open(&path, 0).await.unwrap(), + live: writer.lock().await.live_records(), observers: Vec::new(), origins: WeakOrigins::default(), roots: Roots::default(), @@ -1632,6 +1708,7 @@ mod tests { recovery: Arc::new(Mutex::new(None)), translator: Some(Box::new(CountedTranslation(drains.clone()))), reader: None, + live: LiveRecords::default(), observers: Vec::new(), origins: WeakOrigins::default(), roots: Roots::default(), diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 82e4f634..37dc7135 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -89,7 +89,7 @@ pub(super) struct StartupCursor { } /// Advance each evicted journal range from its previous position, retaining -/// the reader's Pi delta context between advances. Older records are decoded +/// the reader's payload decoder context between advances. Older records are decoded /// only to initialize that context; they are not dispatched again. pub(super) async fn replay_evicted_startup( daemon: &Arc, From 884268340f55d877b597f9f5f55549499a9e2bdd Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Mon, 2 Nov 2026 22:19:27 +0800 Subject: [PATCH 11/19] Separate live pipeline input from journal storage and recovery --- bt-daemon/src/journal.rs | 233 ++++---------------------- bt-daemon/src/journal/codec.rs | 13 ++ bt-daemon/src/journal/live.rs | 70 -------- bt-daemon/src/pipeline/input.rs | 189 +++++++++++++++++++++ bt-daemon/src/pipeline/mod.rs | 1 + bt-daemon/src/pipeline/translation.rs | 223 +++++++++++++++++------- bt-daemon/src/server/ingress.rs | 20 ++- bt-daemon/src/server/maintenance.rs | 1 + bt-daemon/src/server/mod.rs | 2 + bt-daemon/src/server/recovery.rs | 1 + bt-daemon/src/server/sessions.rs | 34 +++- 11 files changed, 447 insertions(+), 340 deletions(-) delete mode 100644 bt-daemon/src/journal/live.rs create mode 100644 bt-daemon/src/pipeline/input.rs diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index 2fd35d6b..2c7cd2c2 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -11,9 +11,9 @@ //! daemon that accepted them has restarted or idle-exited. mod codec; -mod live; +use crate::pipeline::input::SourcePosition; +pub use crate::pipeline::input::{ControlKind, PipelineControl}; pub(crate) use codec::DecoderContext; -pub(crate) use live::LiveRecords; use crate::wire::{BackendAuth, Envelope, RedactedEnvelope, SessionRoute}; use serde::{Deserialize, Serialize}; @@ -227,7 +227,6 @@ pub struct JournalWriter { file: tokio::fs::File, position: u64, payloads: codec::PayloadCache, - live: LiveRecords, } impl JournalWriter { @@ -245,7 +244,6 @@ impl JournalWriter { file, position, payloads: codec::PayloadCache::default(), - live: LiveRecords::default(), }) } @@ -253,10 +251,6 @@ impl JournalWriter { self.position } - pub(crate) fn live_records(&self) -> LiveRecords { - self.live.clone() - } - /// Append one redacted event and confirm its durable write before capture acknowledges it. /// /// The journal is a durability record and is never capped or truncated: @@ -266,14 +260,16 @@ impl JournalWriter { /// byte once, and replay streams records without loading the whole log. #[cfg(test)] pub async fn append(&mut self, env: &Envelope) -> anyhow::Result { - self.append_ordered(env, None).await + self.append_ordered(env, None) + .await + .map(|position| position.through) } pub(crate) async fn append_ordered( &mut self, env: &Envelope, sequence: Option, - ) -> anyhow::Result { + ) -> anyhow::Result { #[derive(Serialize)] struct Captured<'a> { #[serde(flatten)] @@ -284,20 +280,17 @@ impl JournalWriter { } let mut redacted = env.redacted(); let (json, delta) = self.payloads.encode(&env.event, &redacted.payload)?; - let (record_type, original) = if let Some(delta) = delta { - let original = std::mem::replace(&mut redacted.payload, serde_json::to_value(delta)?); - (codec::DELTA_RECORD, Some(original)) + let record_type = if let Some(delta) = delta { + redacted.payload = serde_json::to_value(delta)?; + codec::DELTA_RECORD } else { - ("event", None) + "event" }; let mut line = serde_json::to_vec(&Captured { envelope: &redacted, _bt_record_type: record_type, _bt_capture_sequence: sequence, })?; - if let Some(original) = original { - redacted.payload = original; - } line.push(b'\n'); self.file.write_all(&line).await?; self.file.flush().await?; @@ -306,24 +299,29 @@ impl JournalWriter { self.position += line.len() as u64; self.payloads .record(&env.event, self.position, json.clone()); - self.live.push(live::LiveRecord { + Ok(SourcePosition { before, - entry: JournalRecordEntry { - record: JournalRecord::Event(redacted), - through: self.position, - decoded_bytes: (line.len() as u64) - .max(json.as_ref().map_or(0, |json| json.len() as u64)), - capture_sequence: sequence, - }, - json, - }); - Ok(self.position) + through: self.position, + decoded_bytes: (line.len() as u64) + .max(json.as_ref().map_or(0, |json| json.len() as u64)), + decoder_base: json, + }) } + #[cfg(test)] pub(crate) async fn append_control( &mut self, control: &PipelineControl, ) -> anyhow::Result { + self.append_control_record(control) + .await + .map(|position| position.through) + } + + pub(crate) async fn append_control_record( + &mut self, + control: &PipelineControl, + ) -> anyhow::Result { let mut value = serde_json::to_value(control)?; value["_bt_record_type"] = "pipeline_control".into(); let mut line = serde_json::to_vec(&value)?; @@ -333,17 +331,12 @@ impl JournalWriter { self.file.sync_data().await?; let before = self.position; self.position += line.len() as u64; - self.live.push(live::LiveRecord { + Ok(SourcePosition { before, - entry: JournalRecordEntry { - record: JournalRecord::Control(control.clone()), - through: self.position, - decoded_bytes: line.len() as u64, - capture_sequence: None, - }, - json: None, - }); - Ok(self.position) + through: self.position, + decoded_bytes: line.len() as u64, + decoder_base: None, + }) } } @@ -365,20 +358,6 @@ pub enum JournalRecord { DeliveryCheckpoint { route: SessionRoute, through: u64 }, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ControlKind { - Checkpoint, - Finalize, - Retire, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PipelineControl { - pub kind: ControlKind, - pub bounds: std::collections::HashMap, -} - #[derive(Debug)] pub struct JournalRecordEntry { pub record: JournalRecord, @@ -400,10 +379,6 @@ pub struct JournalReader { line_no: usize, position: u64, decoder: DecoderContext, - through: u64, - needs_seek: bool, - #[cfg(test)] - disk_records: u64, } impl JournalReader { @@ -411,25 +386,6 @@ impl JournalReader { &self.decoder } - /// Consume a complete captured record without touching the journal file. - /// Keep its decoder base solely for checkpoints and later disk recovery. - pub(crate) fn next_live( - &mut self, - live: &LiveRecords, - through: u64, - ) -> Option { - let record = live.take(self.position, through)?; - if let JournalRecord::Event(event) = &record.entry.record { - self.decoder.legacy_pi.clear(); - self.decoder - .payloads - .record(&event.event, record.entry.through, record.json); - } - self.position = record.entry.through; - self.needs_seek = true; - Some(record.entry) - } - /// Resume any source journal at a committed byte offset with its saved /// lossless payload cache (and, for old Pi logs, their legacy message base). pub(crate) async fn resume( @@ -457,10 +413,6 @@ impl JournalReader { line_no: 0, position, decoder: decoder_context, - through, - needs_seek: false, - #[cfg(test)] - disk_records: 0, }) } /// Open a journal for streaming, reading at most `through` bytes. @@ -482,10 +434,6 @@ impl JournalReader { line_no: 0, position: 0, decoder: DecoderContext::default(), - through, - needs_seek: false, - #[cfg(test)] - disk_records: 0, })) } @@ -511,34 +459,12 @@ impl JournalReader { /// The next journal record, including delivery checkpoints written by a /// previous daemon generation. pub async fn next_record(&mut self) -> anyhow::Result> { - if self.position >= self.through { - return Ok(None); - } - if self.needs_seek { - // Live records can skip beyond buffered disk data. Reposition only - // when an evicted or historical record actually needs recovery. - let buffered = self.reader.buffer().len(); - self.reader.consume(buffered); - self.reader - .get_mut() - .get_mut() - .seek(std::io::SeekFrom::Start(self.position)) - .await?; - self.reader - .get_mut() - .set_limit(self.through - self.position); - self.needs_seek = false; - } loop { let mut bytes = Vec::new(); let read = self.reader.read_until(b'\n', &mut bytes).await?; if read == 0 { return Ok(None); } - #[cfg(test)] - { - self.disk_records += 1; - } self.position += read as u64; self.line_no += 1; let line = std::str::from_utf8(&bytes) @@ -626,10 +552,6 @@ impl JournalReader { } pub(crate) fn extend(&mut self, through: u64) { - self.through = through; - if self.needs_seek { - return; - } let buffered = self.reader.buffer().len() as u64; self.reader .get_mut() @@ -950,99 +872,6 @@ mod tests { } } - #[tokio::test] - async fn complete_live_records_bypass_disk_and_delta_expansion_for_every_source() { - for source in crate::translate::Registry::default_agents().sources() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("events.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let live = writer.live_records(); - let mut expected = Vec::new(); - for index in 0..4 { - let mut event = pi_context(vec![], index); - event.source = source.clone(); - event.payload = - serde_json::json!({"history":"shared history".repeat(1024),"index":index}); - writer.append(&event).await.unwrap(); - expected.push(event); - } - let contents = tokio::fs::read_to_string(&path).await.unwrap(); - assert!(contents.contains(codec::DELTA_RECORD)); - let through = writer.position(); - let mut reader = JournalReader::open(&path, through).await.unwrap().unwrap(); - for expected in expected { - let entry = reader - .next_live(&live, through) - .expect("retained live event"); - let JournalRecord::Event(actual) = entry.record else { - panic!("expected event") - }; - assert_eq!(actual.payload, expected.payload, "{source}"); - } - assert!(reader.next_record().await.unwrap().is_none()); - assert_eq!(reader.disk_records, 0, "{source} read back live events"); - } - } - - #[tokio::test] - async fn evicted_records_recover_from_disk_then_rejoin_the_complete_live_feed() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("events.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let live = writer.live_records(); - let mut reader = JournalReader::open(&path, 0).await.unwrap().unwrap(); - for batch in 0..2 { - for index in 0..80 { - let mut event = pi_context(vec![], index); - event.payload = - serde_json::json!({"history":"repeated".repeat(1024),"index":batch*80+index}); - writer.append(&event).await.unwrap(); - } - let through = writer.position(); - reader.extend(through); - for index in 0..80 { - let entry = match reader.next_live(&live, through) { - Some(entry) => entry, - None => reader.next_record().await.unwrap().unwrap(), - }; - let JournalRecord::Event(event) = entry.record else { - panic!("expected event") - }; - assert_eq!(event.payload["index"], batch * 80 + index); - assert_eq!(event.payload["history"], "repeated".repeat(1024)); - } - assert!(reader.next_record().await.unwrap().is_none()); - assert_eq!(reader.disk_records, ((batch + 1) * 16) as u64); - } - } - - #[tokio::test] - async fn live_feed_bounds_decoded_bytes_and_allows_one_oversized_event() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("events.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let live = writer.live_records(); - let mut event = pi_context(vec![], 0); - event.payload = serde_json::json!({"history":"x".repeat(9 * 1024 * 1024)}); - let first = writer.append(&event).await.unwrap(); - let mut reader = JournalReader::open(&path, 0).await.unwrap().unwrap(); - assert!( - reader.next_live(&live, first).is_some(), - "an oversized event must still flow directly" - ); - let second = writer.append(&event).await.unwrap(); - event.payload = serde_json::json!({"history":"x".repeat(3 * 1024 * 1024)}); - let third = writer.append(&event).await.unwrap(); - assert!( - reader.next_live(&live, third).is_none(), - "byte pressure must evict the oversized event" - ); - reader.extend(second); - assert_eq!(reader.next_record().await.unwrap().unwrap().through, second); - assert_eq!(reader.next_live(&live, third).unwrap().through, third); - assert_eq!(reader.disk_records, 1); - } - #[tokio::test] async fn legacy_pi_deltas_recover_and_mix_with_complete_new_events() { let temp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/journal/codec.rs b/bt-daemon/src/journal/codec.rs index 40554eef..d1dced78 100644 --- a/bt-daemon/src/journal/codec.rs +++ b/bt-daemon/src/journal/codec.rs @@ -191,6 +191,19 @@ pub(crate) struct DecoderContext { pub(crate) legacy_pi: Vec, pub(super) payloads: PayloadCache, } +impl DecoderContext { + /// Observe storage metadata independently of the original live payload. + /// This base is saved for recovery; live translation never decodes it. + pub(crate) fn record_live( + &mut self, + event: &str, + position: &crate::pipeline::input::SourcePosition, + ) { + self.legacy_pi.clear(); + self.payloads + .record(event, position.through, position.decoder_base.clone()); + } +} impl<'de> Deserialize<'de> for DecoderContext { fn deserialize>(deserializer: D) -> Result { #[derive(Deserialize)] diff --git a/bt-daemon/src/journal/live.rs b/bt-daemon/src/journal/live.rs deleted file mode 100644 index 45bf3300..00000000 --- a/bt-daemon/src/journal/live.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! Complete durable records awaiting their single source translator. Eviction -//! transfers the recovery obligation to the journal without delaying capture. -use super::JournalRecordEntry; -use parking_lot::Mutex; -use std::{collections::VecDeque, sync::Arc}; - -const RECORD_BUDGET: usize = 64; -const BYTE_BUDGET: u64 = 8 * 1024 * 1024; - -pub(super) struct LiveRecord { - pub before: u64, - pub entry: JournalRecordEntry, - pub json: Option>, -} - -impl LiveRecord { - fn bytes(&self) -> u64 { - self.entry - .decoded_bytes - .saturating_add(self.json.as_ref().map_or(0, |json| json.len() as u64)) - } -} - -#[derive(Default)] -struct Buffer { - records: VecDeque, - bytes: u64, -} - -#[derive(Clone, Default)] -pub(crate) struct LiveRecords(Arc>); - -impl LiveRecords { - pub(super) fn push(&self, record: LiveRecord) { - let mut buffer = self.0.lock(); - let bytes = record.bytes(); - // An oversized event travels alone, just as it does on the SpanOp feed. - while !buffer.records.is_empty() - && (buffer.records.len() >= RECORD_BUDGET - || buffer.bytes.saturating_add(bytes) > BYTE_BUDGET) - { - if let Some(evicted) = buffer.records.pop_front() { - buffer.bytes -= evicted.bytes(); - } - } - buffer.bytes = buffer.bytes.saturating_add(bytes); - buffer.records.push_back(record); - } - - pub(super) fn take(&self, position: u64, through: u64) -> Option { - let mut buffer = self.0.lock(); - // Recovery may already have consumed records whose live copies remained. - while buffer - .records - .front() - .is_some_and(|record| record.entry.through <= position) - { - if let Some(consumed) = buffer.records.pop_front() { - buffer.bytes -= consumed.bytes(); - } - } - let first = buffer.records.front()?; - if first.before != position || first.entry.through > through { - return None; - } - let record = buffer.records.pop_front()?; - buffer.bytes -= record.bytes(); - Some(record) - } -} diff --git a/bt-daemon/src/pipeline/input.rs b/bt-daemon/src/pipeline/input.rs new file mode 100644 index 00000000..61b94a97 --- /dev/null +++ b/bt-daemon/src/pipeline/input.rs @@ -0,0 +1,189 @@ +//! Stage input shared by direct capture and recovery. Storage supplies only +//! position/decoder metadata; original live payloads arrive from capture. +use crate::wire::Envelope; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::{collections::VecDeque, sync::Arc}; + +const RECORD_BUDGET: usize = 64; +const BYTE_BUDGET: u64 = 8 * 1024 * 1024; + +/// Durable input position, returned by the storage side path. Decoder data is +/// used only to checkpoint future recovery, never to construct a live payload. +#[derive(Clone)] +pub(crate) struct SourcePosition { + pub before: u64, + pub through: u64, + pub decoded_bytes: u64, + pub decoder_base: Option>, +} + +pub(crate) enum SourceEvent { + Event(Box), + Control(PipelineControl), + LegacyDeliveryCheckpoint, +} + +pub(crate) struct SourceRecord { + pub event: SourceEvent, + pub position: SourcePosition, +} + +impl SourceRecord { + fn bytes(&self) -> u64 { + self.position.decoded_bytes.saturating_add( + self.position + .decoder_base + .as_ref() + .map_or(0, |json| json.len() as u64), + ) + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ControlKind { + Checkpoint, + Finalize, + Retire, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PipelineControl { + pub kind: ControlKind, + pub bounds: HashMap, +} + +#[derive(Default)] +struct Buffer { + records: VecDeque, + bytes: u64, +} + +#[derive(Clone, Default)] +pub(crate) struct SourceInput(Arc>); + +impl SourceInput { + pub(crate) fn push_event(&self, event: Envelope, position: SourcePosition) { + self.push(SourceRecord { + event: SourceEvent::Event(Box::new(event)), + position, + }); + } + pub(crate) fn is_empty(&self) -> bool { + self.0.lock().records.is_empty() + } + pub(crate) fn push(&self, record: SourceRecord) { + let mut buffer = self.0.lock(); + let bytes = record.bytes(); + // An oversized event travels alone, just as it does on the SpanOp feed. + while !buffer.records.is_empty() + && (buffer.records.len() >= RECORD_BUDGET + || buffer.bytes.saturating_add(bytes) > BYTE_BUDGET) + { + if let Some(evicted) = buffer.records.pop_front() { + buffer.bytes -= evicted.bytes(); + } + } + buffer.bytes = buffer.bytes.saturating_add(bytes); + buffer.records.push_back(record); + } + + pub(crate) fn take(&self, position: u64, through: u64) -> Option { + let mut buffer = self.0.lock(); + // Recovery may already have consumed records whose live copies remained. + while buffer + .records + .front() + .is_some_and(|record| record.position.through <= position) + { + if let Some(consumed) = buffer.records.pop_front() { + buffer.bytes -= consumed.bytes(); + } + } + let first = buffer.records.front()?; + if first.position.before != position || first.position.through > through { + return None; + } + let record = buffer.records.pop_front()?; + buffer.bytes -= record.bytes(); + Some(record) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn event(source: &str, index: usize, bytes: usize) -> Envelope { + serde_json::from_value(json!({ + "source":source, "session_id":"direct", "event":"native", "ts_ms":index, + "payload":{"history":"x".repeat(bytes),"index":index} + })) + .unwrap() + } + + fn position(before: u64, event: &Envelope) -> SourcePosition { + let bytes = serde_json::to_vec(event).unwrap().len() as u64; + SourcePosition { + before, + through: before + bytes, + decoded_bytes: bytes, + decoder_base: None, + } + } + + #[test] + fn every_agent_can_supply_original_events_without_any_journal_adapter() { + for source in crate::translate::Registry::default_agents().sources() { + let input = SourceInput::default(); + let first = event(&source, 0, 16 * 1024); + let first_position = position(0, &first); + let boundary = first_position.through; + input.push_event(first.clone(), first_position); + let second = event(&source, 1, 16 * 1024); + let second_position = position(boundary, &second); + let through = second_position.through; + input.push_event(second.clone(), second_position); + let record = input.take(0, boundary).unwrap(); + let SourceEvent::Event(actual) = record.event else { + panic!("expected event") + }; + assert_eq!(actual.payload, first.payload); + assert!( + input.take(boundary, boundary).is_none(), + "must not outrun the requested boundary" + ); + let record = input.take(boundary, through).unwrap(); + let SourceEvent::Event(actual) = record.event else { + panic!("expected event") + }; + assert_eq!(actual.payload, second.payload); + assert!(input.is_empty()); + } + } + + #[test] + fn oversized_events_flow_directly_but_eviction_keeps_the_input_bounded() { + let input = SourceInput::default(); + let oversized = event("debug", 0, 9 * 1024 * 1024); + let first = position(0, &oversized); + let boundary = first.through; + input.push_event(oversized.clone(), first); + assert!(input.take(0, boundary).is_some()); + input.push_event(oversized.clone(), position(boundary, &oversized)); + let gap_end = position(boundary, &oversized).through; + let next = event("debug", 1, 3 * 1024 * 1024); + let next_position = position(gap_end, &next); + let through = next_position.through; + input.push_event(next, next_position); + assert!( + input.take(boundary, through).is_none(), + "evicted input requires recovery of the gap" + ); + assert!(input.take(gap_end, through).is_some()); + assert!(input.is_empty()); + } +} diff --git a/bt-daemon/src/pipeline/mod.rs b/bt-daemon/src/pipeline/mod.rs index 7a830105..e0765285 100644 --- a/bt-daemon/src/pipeline/mod.rs +++ b/bt-daemon/src/pipeline/mod.rs @@ -1,6 +1,7 @@ //! Typed stage contracts and completion-preserving channel messages. pub(crate) mod delivery; +pub(crate) mod input; pub(crate) mod live; pub(crate) mod plugins; pub(crate) mod prepare; diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 6b792330..81675e00 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -1,4 +1,5 @@ //! One ordered source consumer. Output publication is independent of route delivery. +use super::input::{SourceEvent, SourceInput, SourcePosition, SourceRecord}; use super::journal::{SpanJournal, TranslationCheckpoint}; use super::prepare::Prepare; use super::roots::Roots; @@ -7,9 +8,7 @@ use super::{ Stage, }; use crate::{ - journal::{ - ControlKind, JournalReader, JournalRecord, JournalWriter, LiveRecords, PipelineControl, - }, + journal::{ControlKind, JournalReader, JournalRecord, JournalWriter, PipelineControl}, translate::{AgentTranslator, OriginSnapshot, Registry, SessionCtx, SpanOp, WeakOrigins}, wire::{SessionConfig, SessionRoute}, }; @@ -108,6 +107,7 @@ pub(crate) struct TranslationOptions { pub data_dir: PathBuf, pub events: Arc>, pub event_path: PathBuf, + pub input: SourceInput, pub registry: Arc, pub correlation: Arc, pub correlation_changed: Arc, @@ -120,7 +120,6 @@ impl Translation { crate::ids::session_storage_id(&options.source, &options.session_id) )); let journal = SpanJournal::open(&path).await?; - let live = options.events.lock().await.live_records(); let committed = journal.translated_through().await?; let progress = Arc::new(Progress { requested: AtomicU64::new(committed), @@ -143,7 +142,7 @@ impl Translation { recovery: recovery.clone(), translator: None, reader: None, - live, + decoder: Default::default(), observers: Vec::new(), origins: WeakOrigins::default(), roots: Roots::default(), @@ -285,7 +284,7 @@ struct Producer { recovery: Arc>>, translator: Option>, reader: Option, - live: LiveRecords, + decoder: crate::journal::DecoderContext, observers: Vec, origins: WeakOrigins, roots: Roots, @@ -456,24 +455,21 @@ impl Producer { &mut self, input: u64, ) -> anyhow::Result> { - if !self.checkpoint_safe || self.translator.is_none() || self.reader.is_none() { + if !self.checkpoint_safe || self.translator.is_none() { return Ok(None); } let translator = self .translator .take() .ok_or_else(|| anyhow::anyhow!("translator is missing"))?; - let reader = self - .reader - .take() - .ok_or_else(|| anyhow::anyhow!("source reader is missing"))?; + let decoder = self.decoder.clone(); let capture = self.capture.clone(); let bounds = self.bounds.clone(); let templates = self.templates.clone(); let version = self.options.version.clone(); // Context may include a long active conversation. Its encoding and // compression must not monopolize a runtime thread serving hook IPC. - let (translator, reader, checkpoint) = tokio::task::spawn_blocking(move || { + let (translator, checkpoint) = tokio::task::spawn_blocking(move || { let checkpoint = (|| -> anyhow::Result> { let Some(state) = translator.save_state()? else { return Ok(None); @@ -483,7 +479,7 @@ impl Producer { translator: serde_json::value::RawValue::from_string(String::from_utf8( state, )?)?, - decoder: reader.decoder_context().clone(), + decoder, capture, bounds, templates, @@ -497,11 +493,10 @@ impl Producer { state, })) })(); - (translator, reader, checkpoint) + (translator, checkpoint) }) .await?; self.translator = Some(translator); - self.reader = Some(reader); checkpoint } @@ -564,13 +559,16 @@ impl Producer { bounds.insert(path.clone(), through); } let control = PipelineControl { kind, bounds }; - let through = self - .options - .events - .lock() - .await - .append_control(&control) - .await?; + let through = { + let mut writer = self.options.events.lock().await; + let position = writer.append_control_record(&control).await?; + let through = position.through; + self.options.input.push(SourceRecord { + event: SourceEvent::Control(control), + position, + }); + through + }; self.progress .requested .fetch_max(through, Ordering::Relaxed); @@ -600,6 +598,7 @@ impl Producer { )?, ); self.position = 0; + self.decoder = Default::default(); self.checkpoint_position = 0; self.checkpoint_events = 0; self.checkpoint_work_bytes = 0; @@ -627,11 +626,12 @@ impl Producer { &self.options.event_path, input, self.committed, - context.decoder, + context.decoder.clone(), ) .await?; self.translator = Some(translator); self.reader = Some(reader); + self.decoder = context.decoder; self.capture = context.capture; self.bounds = context.bounds; self.templates = context.templates; @@ -648,10 +648,65 @@ impl Producer { } } } - self.reader = JournalReader::open(&self.options.event_path, self.committed).await?; + self.reader = if self.committed > 0 { + JournalReader::open(&self.options.event_path, self.committed).await? + } else { + None + }; Ok(()) } + async fn next_input(&mut self, through: u64) -> anyhow::Result> { + if let Some(record) = self.options.input.take(self.position, through) { + if let SourceEvent::Event(event) = &record.event { + self.decoder.record_live(&event.event, &record.position); + } + // A future gap resumes from this logical position. The live path + // does not open, seek, read, or pass payloads through a journal reader. + self.reader = None; + return Ok(Some(record)); + } + if self.position >= through { + return Ok(None); + } + if self.reader.is_none() { + self.reader = Some( + JournalReader::resume( + &self.options.event_path, + self.position, + through, + self.decoder.clone(), + ) + .await?, + ); + } + let reader = self + .reader + .as_mut() + .ok_or_else(|| anyhow::anyhow!("recovery reader is missing"))?; + reader.extend(through); + let Some(entry) = reader.next_record().await? else { + return Ok(None); + }; + self.decoder = reader.decoder_context().clone(); + let event = match entry.record { + JournalRecord::Event(event) => { + SourceEvent::Event(Box::new(crate::journal::envelope_from_redacted(event))) + } + JournalRecord::Control(control) => SourceEvent::Control(control), + JournalRecord::DeliveryCheckpoint { .. } => SourceEvent::LegacyDeliveryCheckpoint, + }; + Ok(Some(SourceRecord { + event, + position: SourcePosition { + before: self.position, + through: entry.through, + decoded_bytes: entry.decoded_bytes, + decoder_base: None, + }, + })) + } + async fn advance(&mut self, through: u64) -> anyhow::Result<()> { anyhow::ensure!( matches!(self.state, State::Processing), @@ -666,35 +721,16 @@ impl Producer { if self.position >= through { return Ok(()); } - let reader = self - .reader - .as_mut() - .ok_or_else(|| anyhow::anyhow!("event journal is missing"))?; - reader.extend(through); self.phase = Boundary::Journal; - loop { - let reader = self - .reader - .as_mut() - .ok_or_else(|| anyhow::anyhow!("event journal is missing"))?; - // Captured records stay complete in memory. Only an evicted or - // historical gap needs a journal read to reconstruct source state. - let entry = match reader.next_live(&self.live, through) { - Some(entry) => Some(entry), - None => reader.next_record().await?, - }; - let Some(entry) = entry else { - break; - }; - let input = entry.through; - let observation = !matches!(&entry.record,JournalRecord::Control(control) if matches!(control.kind,ControlKind::Retire)); + while let Some(entry) = self.next_input(through).await? { + let input = entry.position.through; + let observation = !matches!(&entry.event,SourceEvent::Control(control) if matches!(control.kind,ControlKind::Retire)); let ctx = SessionCtx { session_id: self.options.session_id.clone(), config: None, }; - let (first, capture) = match entry.record { - JournalRecord::Event(event) => { - let mut env = crate::journal::envelope_from_redacted(event); + let (first, capture) = match entry.event { + SourceEvent::Event(mut env) => { if let Some(route) = &env.route { self.journal.register(route).await?; // Discovery needs subscriptions and transcript references, @@ -762,7 +798,7 @@ impl Producer { Some(capture), ) } - JournalRecord::Control(control) => { + SourceEvent::Control(control) => { // Retirement exports defensive terminal updates but does not // close native state when rebuilding for later captured work. if input <= self.committed && matches!(control.kind, ControlKind::Retire) { @@ -787,7 +823,7 @@ impl Producer { }; (ops, self.capture.clone()) } - JournalRecord::DeliveryCheckpoint { .. } => { + SourceEvent::LegacyDeliveryCheckpoint => { if input > self.committed { self.journal.begin(input, observation).await?; self.journal.commit(input).await?; @@ -800,7 +836,7 @@ impl Producer { }; self.publication = Some(Publication { input, - decoded_bytes: entry.decoded_bytes, + decoded_bytes: entry.position.decoded_bytes, ordinal: 0, observation, capture, @@ -1027,6 +1063,7 @@ mod tests { data_dir: data_dir.into(), events, event_path: crate::journal::source_journal_path(data_dir, source, "checkpoint"), + input: SourceInput::default(), registry: Arc::new(Registry::default_agents()), correlation: Arc::default(), correlation_changed: Arc::default(), @@ -1040,17 +1077,17 @@ mod tests { let writer = Arc::new(tokio::sync::Mutex::new( JournalWriter::open_path(&path).await.unwrap(), )); - let translation = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) - .await - .unwrap(); + let options = resume_options(tmp.path(), "debug", writer.clone()); + let input = options.input.clone(); + let translation = Translation::spawn(options).await.unwrap(); let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ "source":"debug", "session_id":"checkpoint", "event":"event", "ts_ms":1, "payload":{"history":"shared history".repeat(1024),"index":0} })) .unwrap(); - writer.lock().await.append(&event).await.unwrap(); + capture(&writer, &input, &event).await; event.payload["index"] = 1.into(); - let through = writer.lock().await.append(&event).await.unwrap(); + let through = capture(&writer, &input, &event).await; let original = tokio::fs::read(&path).await.unwrap(); assert!(String::from_utf8_lossy(&original).contains("payload_delta_v1")); // A journal read would fail. The complete captured records are still @@ -1096,6 +1133,72 @@ mod tests { resumed.shutdown().await; } + async fn capture( + writer: &Arc>, + input: &SourceInput, + event: &crate::wire::Envelope, + ) -> u64 { + let mut writer = writer.lock().await; + let position = writer.append_ordered(event, None).await.unwrap(); + let through = position.through; + input.push_event(event.clone(), position); + through + } + + #[tokio::test] + async fn evicted_input_recovers_only_the_gap_then_rejoins_direct_capture() { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let options = resume_options(tmp.path(), "debug", writer.clone()); + let input = options.input.clone(); + let translation = Translation::spawn(options).await.unwrap(); + let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ + "source":"debug", "session_id":"checkpoint", "event":"event", "ts_ms":1, + "payload":{"history":"shared".repeat(1024),"index":0} + })) + .unwrap(); + for batch in 0..2 { + let mut recovery_end = 0; + let mut through = 0; + for index in 0..80 { + event.payload["index"] = (batch * 80 + index).into(); + through = capture(&writer, &input, &event).await; + if index == 15 { + recovery_end = through; + } + } + // Only the first 16 records were evicted. Poison the retained + // live suffix so any attempt to read it through storage fails. + let mut stored = tokio::fs::read(&path).await.unwrap(); + for byte in &mut stored[recovery_end as usize..through as usize] { + if *byte != b'\n' { + *byte = b'x'; + } + } + tokio::fs::write(&path, stored).await.unwrap(); + translation.advance(through).await.unwrap(); + translation.barrier().await.unwrap(); + assert!(input.is_empty()); + } + let rows = translation.journal.read_after(0, 256).await.unwrap(); + let inputs = rows + .iter() + .filter_map(|(_, op)| { + let (SpanOp::Insert(row) | SpanOp::Merge(row)) = op; + row.input.as_ref() + }) + .collect::>(); + assert_eq!(inputs.len(), 160); + for (index, payload) in inputs.into_iter().enumerate() { + assert_eq!(payload["index"], index); + assert_eq!(payload["history"], event.payload["history"]); + } + translation.shutdown().await; + } + struct BlockedCheckpoint { entered: Arc, release: std::sync::mpsc::Receiver<()>, @@ -1138,7 +1241,7 @@ mod tests { release: resume, })), reader: JournalReader::open(&path, 0).await.unwrap(), - live: writer.lock().await.live_records(), + decoder: Default::default(), observers: Vec::new(), origins: WeakOrigins::default(), roots: Roots::default(), @@ -1518,6 +1621,7 @@ mod tests { JournalWriter::open_path(&event_path).await.unwrap(), )), event_path, + input: SourceInput::default(), registry: Arc::new(Registry::default_agents()), correlation: Arc::new(crate::correlation::CorrelationRegistry::default()), correlation_changed: Arc::new(Notify::new()), @@ -1548,6 +1652,7 @@ mod tests { JournalWriter::open_path(&event_path).await.unwrap(), )), event_path, + input: SourceInput::default(), registry: Arc::new(Registry::default_agents()), correlation: correlation.clone(), correlation_changed: Arc::new(Notify::new()), @@ -1643,6 +1748,7 @@ mod tests { JournalWriter::open_path(&event_path).await.unwrap(), )), event_path, + input: SourceInput::default(), registry: Arc::new(Registry::default_agents()), correlation: Arc::new(crate::correlation::CorrelationRegistry::default()), correlation_changed: Arc::new(Notify::new()), @@ -1698,6 +1804,7 @@ mod tests { JournalWriter::open_path(&event_path).await.unwrap(), )), event_path, + input: SourceInput::default(), registry: Arc::new(Registry::default_agents()), correlation: Arc::new(crate::correlation::CorrelationRegistry::default()), correlation_changed: Arc::new(Notify::new()), @@ -1708,7 +1815,7 @@ mod tests { recovery: Arc::new(Mutex::new(None)), translator: Some(Box::new(CountedTranslation(drains.clone()))), reader: None, - live: LiveRecords::default(), + decoder: Default::default(), observers: Vec::new(), origins: WeakOrigins::default(), roots: Roots::default(), diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 37dc7135..041177e3 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -550,11 +550,25 @@ impl Daemon { /// Caller holds the source-session capture lock; frozen observations enter /// the same durable queue as hooks without reacquiring that lock. - pub(super) async fn journal_and_enqueue(&self, mut env: Envelope) -> Result { - let (replay_through, journal_through) = self - .append_to_journal(&mut env) + pub(super) async fn journal_and_enqueue(&self, env: Envelope) -> Result { + let writer = self + .journal_writer_for(&env.source, &env.session_id) .await .map_err(|error| format!("journal failed: {error}"))?; + let input = self.source_input_for(&env.source, &env.session_id); + let (replay_through, journal_through) = { + let mut writer = writer.lock().await; + // Storage is a side branch: it receives a copy and returns only + // durable position/context metadata. The stage receives the + // original complete event directly from capture. + let position = self + .append_to_journal(&env, &mut writer) + .await + .map_err(|error| format!("journal failed: {error}"))?; + let bounds = (position.before, position.through); + input.push_event(env.clone(), position); + bounds + }; let event = PendingEvent { env, replay_through, diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index d5aae34d..abccb6f4 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -379,6 +379,7 @@ pub(super) async fn drain_all(daemon: &Arc) { daemon.sessions.lock().clear(); daemon.translations.lock().clear(); daemon.journals.lock().clear(); + daemon.source_inputs.lock().clear(); let (reply, done) = oneshot::channel(); if daemon .ingress_tx diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index 93d7184f..949fba45 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -116,6 +116,7 @@ pub struct Daemon { /// capture, which must never inherit actor or sink backpressure. dispatch_locks: Mutex>>>, journals: Mutex>>>, + source_inputs: Mutex>, claude_transcripts: Mutex>, managed_run_sessions: Mutex>>, auth_errors: Mutex>, @@ -172,6 +173,7 @@ impl Daemon { session_locks: Mutex::new(HashMap::new()), dispatch_locks: Mutex::new(HashMap::new()), journals: Mutex::new(HashMap::new()), + source_inputs: Mutex::new(HashMap::new()), claude_transcripts: Mutex::new(HashMap::new()), managed_run_sessions: Mutex::new(HashMap::new()), auth_errors: Mutex::new(HashMap::new()), diff --git a/bt-daemon/src/server/recovery.rs b/bt-daemon/src/server/recovery.rs index 8aa35c8f..738f30ea 100644 --- a/bt-daemon/src/server/recovery.rs +++ b/bt-daemon/src/server/recovery.rs @@ -359,6 +359,7 @@ mod tests { data_dir: tmp.path().into(), events: writer.clone(), event_path: raw.clone(), + input: crate::pipeline::input::SourceInput::default(), registry: Arc::new(Registry::default_agents()), correlation: Arc::default(), correlation_changed: Arc::default(), diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index 09156367..d0a720ec 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -64,6 +64,7 @@ impl Daemon { data_dir: self.data_dir.clone(), events: journal.clone(), event_path: event_path.clone(), + input: self.source_input_for(&env.source, &env.session_id), registry: self.translators.clone(), correlation: self.correlation.clone(), correlation_changed: self.correlation_changed.clone(), @@ -222,6 +223,14 @@ impl Daemon { candidate.source != key.source || candidate.session_id != key.session_id }); self.journals.lock().remove(&storage_key); + let mut inputs = self.source_inputs.lock(); + if inputs + .get(&storage_key) + .is_some_and(|input| input.is_empty()) + { + inputs.remove(&storage_key); + } + drop(inputs); self.translations.lock().remove(&storage_key); { let mut locks = self.session_locks.lock(); @@ -272,11 +281,23 @@ impl Daemon { .clone()) } - pub(super) async fn append_to_journal(&self, env: &mut Envelope) -> anyhow::Result<(u64, u64)> { - let writer = self - .journal_writer_for(&env.source, &env.session_id) - .await?; - let mut writer = writer.lock().await; + pub(super) fn source_input_for( + &self, + source: &str, + session_id: &str, + ) -> crate::pipeline::input::SourceInput { + self.source_inputs + .lock() + .entry(crate::ids::session_namespace(source, session_id)) + .or_default() + .clone() + } + + pub(super) async fn append_to_journal( + &self, + env: &Envelope, + writer: &mut JournalWriter, + ) -> anyhow::Result { let before = writer.position(); if let Some(cutoffs) = self.startup_cutoffs.lock().as_mut() { let path = journal::source_journal_path(&self.data_dir, &env.source, &env.session_id); @@ -285,8 +306,7 @@ impl Daemon { // Related hooks cannot begin until their predecessor's durable append // is acknowledged. Concurrent sessions need no stronger total order. let sequence = self.capture_sequence.fetch_add(1, Ordering::Relaxed); - let through = writer.append_ordered(env, Some(sequence)).await?; - Ok((before, through)) + writer.append_ordered(env, Some(sequence)).await } pub(super) fn session_lock( From 571cea70dcc8aa2da75ad309579d9f934d446a88 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 3 Nov 2026 00:22:50 +0800 Subject: [PATCH 12/19] Preserve durable capture boundaries and causal recovery order --- bt-daemon/src/journal.rs | 468 ++++++++++++++---- bt-daemon/src/journal/codec.rs | 10 + bt-daemon/src/pipeline/journal.rs | 61 ++- bt-daemon/src/pipeline/translation.rs | 333 +++++++++++-- bt-daemon/src/server/ingress.rs | 257 ++++++---- bt-daemon/src/server/maintenance.rs | 123 ++++- bt-daemon/src/server/mod.rs | 12 +- bt-daemon/src/server/recovery.rs | 5 +- bt-daemon/src/server/sessions.rs | 68 ++- .../tests/support/translation_streams.rs | 159 ++++++ bt-daemon/tests/translator_checkpoints.rs | 141 +----- 11 files changed, 1244 insertions(+), 393 deletions(-) create mode 100644 bt-daemon/tests/support/translation_streams.rs diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index 2c7cd2c2..acad9326 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -227,6 +227,33 @@ pub struct JournalWriter { file: tokio::fs::File, position: u64, payloads: codec::PayloadCache, + failed_write: bool, +} + +/// Ignore only an unterminated trailing write. Complete records, including +/// malformed ones, remain visible so corruption cannot silently discard data. +async fn complete_position(file: &mut tokio::fs::File) -> std::io::Result { + let len = file.metadata().await?.len(); + if len == 0 { + return Ok(0); + } + file.seek(std::io::SeekFrom::Start(len - 1)).await?; + if file.read_u8().await? == b'\n' { + return Ok(len); + } + let mut end = len; + let mut buffer = vec![0; 8192]; + while end > 0 { + let start = end.saturating_sub(buffer.len() as u64); + let bytes = &mut buffer[..(end - start) as usize]; + file.seek(std::io::SeekFrom::Start(start)).await?; + file.read_exact(bytes).await?; + if let Some(index) = bytes.iter().rposition(|byte| *byte == b'\n') { + return Ok(start + index as u64 + 1); + } + end = start; + } + Ok(0) } impl JournalWriter { @@ -234,16 +261,22 @@ impl JournalWriter { if let Some(dir) = path.parent() { tokio::fs::create_dir_all(dir).await?; } - let file = tokio::fs::OpenOptions::new() + let mut file = tokio::fs::OpenOptions::new() .create(true) + .read(true) .append(true) .open(path) .await?; - let position = file.metadata().await?.len(); + let position = complete_position(&mut file).await?; + if position < file.metadata().await?.len() { + file.set_len(position).await?; + } + file.sync_data().await?; Ok(Self { file, position, payloads: codec::PayloadCache::default(), + failed_write: false, }) } @@ -251,10 +284,31 @@ impl JournalWriter { self.position } + async fn append_bytes(&mut self, line: &[u8]) -> anyhow::Result { + if self.failed_write { + self.file.set_len(self.position).await?; + self.file.sync_data().await?; + self.failed_write = false; + } + if let Err(error) = async { + self.file.write_all(line).await?; + self.file.flush().await?; + self.file.sync_data().await + } + .await + { + self.failed_write = true; + return Err(error.into()); + } + let before = self.position; + self.position += line.len() as u64; + Ok(before) + } + /// Append one redacted event and confirm its durable write before capture acknowledges it. /// - /// The journal is a durability record and is never capped or truncated: - /// dropping entries would silently cost recovery fidelity. Its size is + /// Complete journal records are never capped or truncated. An unfinished + /// trailing write is discarded on reopen, before new appends. Its size is /// managed by age-based GC. Native event payloads are preserved, including /// repeated message snapshots. External transcript mirrors capture each /// byte once, and replay streams records without loading the whole log. @@ -292,11 +346,7 @@ impl JournalWriter { _bt_capture_sequence: sequence, })?; line.push(b'\n'); - self.file.write_all(&line).await?; - self.file.flush().await?; - self.file.sync_data().await?; - let before = self.position; - self.position += line.len() as u64; + let before = self.append_bytes(&line).await?; self.payloads .record(&env.event, self.position, json.clone()); Ok(SourcePosition { @@ -326,11 +376,7 @@ impl JournalWriter { value["_bt_record_type"] = "pipeline_control".into(); let mut line = serde_json::to_vec(&value)?; line.push(b'\n'); - self.file.write_all(&line).await?; - self.file.flush().await?; - self.file.sync_data().await?; - let before = self.position; - self.position += line.len() as u64; + let before = self.append_bytes(&line).await?; Ok(SourcePosition { before, through: self.position, @@ -382,10 +428,16 @@ pub struct JournalReader { } impl JournalReader { + #[cfg(test)] pub(crate) fn decoder_context(&self) -> &DecoderContext { &self.decoder } + /// The source consumer owns decoding state between recovery reads. + pub(crate) fn swap_decoder_context(&mut self, context: &mut DecoderContext) { + std::mem::swap(&mut self.decoder, context); + } + /// Resume any source journal at a committed byte offset with its saved /// lossless payload cache (and, for old Pi logs, their legacy message base). pub(crate) async fn resume( @@ -440,10 +492,10 @@ impl JournalReader { /// The journal's current length, which is the bound a session created now /// should replay through. A missing journal replays nothing. pub async fn recorded_len(path: &Path) -> u64 { - tokio::fs::metadata(path) - .await - .map(|meta| meta.len()) - .unwrap_or(0) + let Ok(mut file) = tokio::fs::File::open(path).await else { + return 0; + }; + complete_position(&mut file).await.unwrap_or(0) } /// The next entry, or `None` at end of file. @@ -467,87 +519,25 @@ impl JournalReader { } self.position += read as u64; self.line_no += 1; - let line = std::str::from_utf8(&bytes) - .map_err(|error| { - anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) - })? - .trim(); - if line.is_empty() { - continue; + anyhow::ensure!( + bytes.last() == Some(&b'\n'), + "unterminated source journal record" + ); + let path = self.path.clone(); + let line_no = self.line_no; + let position = self.position; + let mut decoder = std::mem::take(&mut self.decoder); + // Expanding and parsing a large history can be CPU-heavy even + // when file reads are async. Never run it on the hook IPC thread. + let (decoder, decoded) = tokio::task::spawn_blocking(move || { + let decoded = decode_record(&bytes, &path, line_no, position, &mut decoder); + (decoder, decoded) + }) + .await?; + self.decoder = decoder; + if let Some(entry) = decoded? { + return Ok(Some(entry)); } - let value: serde_json::Value = serde_json::from_str(line).map_err(|error| { - anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) - })?; - let capture_sequence = value - .get("_bt_capture_sequence") - .and_then(serde_json::Value::as_u64); - let mut decoded_bytes = read as u64; - let record = if value - .get("_bt_record_type") - .and_then(serde_json::Value::as_str) - == Some("delivery_checkpoint") - { - let checkpoint: DeliveryCheckpointRecord = - serde_json::from_value(value).map_err(|error| { - anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) - })?; - JournalRecord::DeliveryCheckpoint { - route: checkpoint.route, - through: checkpoint.through, - } - } else if value - .get("_bt_record_type") - .and_then(serde_json::Value::as_str) - == Some("pipeline_control") - { - JournalRecord::Control(serde_json::from_value(value)?) - } else { - let record_type = value - .get("_bt_record_type") - .and_then(serde_json::Value::as_str) - .map(str::to_owned); - let mut event: RedactedEnvelope = - serde_json::from_value(value).map_err(|error| { - anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) - })?; - match record_type.as_deref() { - Some("event") | Some(codec::DELTA_RECORD) => { - self.decoder.legacy_pi.clear(); - let json = if record_type.as_deref() == Some(codec::DELTA_RECORD) { - let (payload, json) = - self.decoder.payloads.decode(&event.event, event.payload)?; - decoded_bytes = decoded_bytes.max(json.len() as u64); - event.payload = payload; - Some(json) - } else { - codec::cache_json(&event.event, &event.payload)? - }; - self.decoder - .payloads - .record(&event.event, self.position, json); - } - None => { - self.decoder.payloads = codec::PayloadCache::default(); - expand_legacy_pi_payload(&mut event, &mut self.decoder.legacy_pi).map_err( - |error| { - anyhow::anyhow!( - "journal {}:{}: {error}", - self.path.display(), - self.line_no - ) - }, - )?; - } - Some(kind) => anyhow::bail!("unsupported journal record type {kind:?}"), - } - JournalRecord::Event(event) - }; - return Ok(Some(JournalRecordEntry { - record, - through: self.position, - decoded_bytes, - capture_sequence, - })); } } @@ -562,6 +552,9 @@ impl JournalReader { /// journal being recovered. Older journals contain no checkpoints and /// therefore retain their existing replay behavior. pub async fn acknowledged_through(path: &Path, through: u64, route: &SessionRoute) -> u64 { + if through == 0 { + return 0; + } let Ok(Some(mut reader)) = Self::open(path, through).await else { return 0; }; @@ -581,6 +574,86 @@ impl JournalReader { } } +fn decode_record( + bytes: &[u8], + path: &Path, + line_no: usize, + position: u64, + decoder: &mut DecoderContext, +) -> anyhow::Result> { + let line = std::str::from_utf8(bytes) + .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))? + .trim(); + if line.is_empty() { + return Ok(None); + } + let value: serde_json::Value = serde_json::from_str(line) + .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))?; + let capture_sequence = value + .get("_bt_capture_sequence") + .and_then(serde_json::Value::as_u64); + let mut decoded_bytes = bytes.len() as u64; + let record = if value + .get("_bt_record_type") + .and_then(serde_json::Value::as_str) + == Some("delivery_checkpoint") + { + let checkpoint: DeliveryCheckpointRecord = serde_json::from_value(value) + .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))?; + JournalRecord::DeliveryCheckpoint { + route: checkpoint.route, + through: checkpoint.through, + } + } else if value + .get("_bt_record_type") + .and_then(serde_json::Value::as_str) + == Some("pipeline_control") + { + JournalRecord::Control(serde_json::from_value(value)?) + } else { + let record_type = value + .get("_bt_record_type") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let mut event: RedactedEnvelope = serde_json::from_value(value) + .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))?; + match record_type.as_deref() { + Some("event") | Some(codec::DELTA_RECORD) => { + decoder.legacy_pi.clear(); + let json = if record_type.as_deref() == Some(codec::DELTA_RECORD) { + let (payload, json) = decoder.payloads.decode(&event.event, event.payload)?; + decoded_bytes = decoded_bytes.max(json.len() as u64); + event.payload = payload; + Some(json) + } else { + #[derive(Deserialize)] + struct StoredPayload<'a> { + #[serde(borrow)] + payload: &'a serde_json::value::RawValue, + } + let stored: StoredPayload<'_> = serde_json::from_str(line)?; + codec::cache_serialized(&event.event, stored.payload.get()) + }; + decoder.payloads.record(&event.event, position, json); + } + None => { + decoder.payloads = codec::PayloadCache::default(); + expand_legacy_pi_payload(&mut event, &mut decoder.legacy_pi).map_err(|error| { + anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no) + })?; + } + Some(kind) => anyhow::bail!("unsupported journal record type {kind:?}"), + } + JournalRecord::Event(event) + }; + Ok(Some(JournalRecordEntry { + record, + through: position, + decoded_bytes, + capture_sequence, + })) +} + const LEGACY_PI_MESSAGES_DELTA: &str = "_bt_messages_delta"; // Read compatibility for journals written before native payload preservation. @@ -747,6 +820,13 @@ pub fn envelope_from_redacted(r: RedactedEnvelope) -> Envelope { #[cfg(test)] mod tests { use super::*; + mod streams { + use crate as bt_daemon; + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/translation_streams.rs" + )); + } fn pi_context(messages: Vec, ts_ms: i64) -> Envelope { Envelope { @@ -764,6 +844,198 @@ mod tests { } } + #[tokio::test] + async fn compressed_recovery_matches_live_native_translation_for_every_source() { + use crate::translate::{AgentTranslator, Registry, SessionCtx, SpanOp}; + fn collect( + translator: &mut dyn AgentTranslator, + env: &Envelope, + ctx: &SessionCtx, + ) -> Vec { + let mut ops = translator.handle(env, ctx).unwrap(); + while let Some(batch) = translator.drain_pending(ctx).unwrap() { + ops.extend(batch); + } + comparable(ops) + } + fn comparable(ops: Vec) -> Vec { + let mut values = ops + .into_iter() + .map(|op| { + let (SpanOp::Insert(row) | SpanOp::Merge(row)) = &op; + serde_json::to_string(&( + &op, + &row.turn_span_id, + row.is_turn, + &row.late_merge_key, + &row.late_merge_version, + )) + .unwrap() + }) + .collect::>(); + values.sort(); + values + } + let registry = Registry::default_agents(); + let mut covered = Vec::new(); + for (source, mut events, _fixtures) in streams::native_streams() { + covered.push(source.clone()); + // A retried first hook is valid native input and guarantees an + // actual delta for every agent, even a short unique-kind fixture. + events.insert(0, events[0].clone()); + for event in &mut events { + event.payload["journal_codec_padding"] = + serde_json::json!("é📚shared history".repeat(1024)); + } + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let mut tail = 0; + for event in &events { + tail = writer.append(event).await.unwrap(); + } + assert!(tokio::fs::read_to_string(&path) + .await + .unwrap() + .contains(codec::DELTA_RECORD)); + let session = events[0].session_id.clone(); + let ctx = SessionCtx { + session_id: session.clone(), + config: None, + }; + let mut live = registry.create(&source, &session); + let mut recovered = registry.create(&source, &session); + let mut reader = JournalReader::open(&path, tail).await.unwrap().unwrap(); + for (index, original) in events.iter().enumerate() { + let replayed = reader.next_entry().await.unwrap().unwrap(); + assert_eq!( + serde_json::to_value(&replayed).unwrap(), + serde_json::to_value(original.redacted()).unwrap(), + "{source} payload {index}" + ); + let replayed = envelope_from_redacted(replayed); + assert_eq!( + collect(live.as_mut(), original, &ctx), + collect(recovered.as_mut(), &replayed, &ctx), + "{source} event {index}" + ); + // Restart the decoder at every boundary, using only its saved + // cache. No previous record may be needed to expand the next. + let saved = serde_json::to_vec(reader.decoder_context()).unwrap(); + reader = JournalReader::resume( + &path, + reader.position, + tail, + serde_json::from_slice(&saved).unwrap(), + ) + .await + .unwrap(); + } + assert!(reader.next_entry().await.unwrap().is_none()); + assert_eq!( + comparable(live.finalize(&ctx).unwrap()), + comparable(recovered.finalize(&ctx).unwrap()), + "{source} finalization" + ); + } + covered.sort(); + assert_eq!(covered, registry.sources()); + } + + #[tokio::test] + async fn delta_recovery_uses_original_json_bytes_not_reader_serialization() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("original-json.ndjson"); + // Key order and number formatting may differ across serde feature + // sets/versions. Offsets always refer to the writer's exact JSON. + let raw = format!(r#"{{"z":"{}","a":1e-3}}"#, "x".repeat(4096)); + let first = format!("{{\"_bt_record_type\":\"event\",\"source\":\"debug\",\"session_id\":\"bytes\",\"event\":\"native\",\"ts_ms\":1,\"payload\":{raw}}}\n"); + let prefix = raw.find("1e-3").unwrap(); + let delta = serde_json::json!({ + "_bt_record_type":codec::DELTA_RECORD, "source":"debug", "session_id":"bytes", "event":"native", "ts_ms":2, + "payload":{"base":first.len(),"prefix":prefix,"suffix":raw.len()-prefix-4,"insert":"2e-3"} + }); + tokio::fs::write(&path, format!("{first}{delta}\n")) + .await + .unwrap(); + let through = JournalReader::recorded_len(&path).await; + let mut reader = JournalReader::open(&path, through).await.unwrap().unwrap(); + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload["a"], + 0.001 + ); + let next = reader.next_entry().await.unwrap().unwrap(); + assert_eq!(next.payload["a"], 0.002); + assert_eq!(next.payload["z"], "x".repeat(4096)); + } + + #[tokio::test] + async fn failed_append_retry_preserves_the_durable_prefix_and_delta_base() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("retry.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let first = pi_context( + vec![serde_json::json!({"content":"history".repeat(1024)})], + 1, + ); + let through = writer.append(&first).await.unwrap(); + // Model a failed write which left OS-visible bytes but did not finish + // durability or update the codec's acknowledged base. + writer.file.write_all(b"{unfinished").await.unwrap(); + writer.file.flush().await.unwrap(); + writer.failed_write = true; + let mut second = first.clone(); + second.payload["tail"] = "after retry".into(); + let position = writer.append_ordered(&second, None).await.unwrap(); + assert_eq!(position.before, through); + let stored = tokio::fs::read_to_string(&path).await.unwrap(); + assert!(stored.contains(codec::DELTA_RECORD)); + let mut reader = JournalReader::open(&path, position.through) + .await + .unwrap() + .unwrap(); + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + first.payload + ); + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + second.payload + ); + assert!(reader.next_entry().await.unwrap().is_none()); + } + + #[tokio::test] + async fn torn_tail_is_excluded_and_removed_before_new_appends() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let event = pi_context(vec![serde_json::json!({"content":"complete"})], 1); + let through = writer.append(&event).await.unwrap(); + drop(writer); + let mut stored = tokio::fs::read(&path).await.unwrap(); + stored.extend_from_slice(b"{\"source\":\"pi\",\"payload\":"); + tokio::fs::write(&path, stored).await.unwrap(); + assert_eq!( + JournalReader::recorded_len(&path).await, + through, + "a torn trailing write is outside the recovery snapshot" + ); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + assert_eq!(writer.position(), through); + let tail = writer.append(&event).await.unwrap(); + let mut reader = JournalReader::open(&path, tail).await.unwrap().unwrap(); + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + event.payload + ); + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + event.payload + ); + assert!(reader.next_entry().await.unwrap().is_none()); + } + #[tokio::test] async fn every_source_journals_complete_payloads_in_event_order() { for source in crate::translate::Registry::default_agents().sources() { diff --git a/bt-daemon/src/journal/codec.rs b/bt-daemon/src/journal/codec.rs index d1dced78..fa039695 100644 --- a/bt-daemon/src/journal/codec.rs +++ b/bt-daemon/src/journal/codec.rs @@ -69,6 +69,16 @@ pub(super) fn cache_json(event: &str, payload: &Value) -> anyhow::Result Option> { + if json.len() < MIN_BYTES || json.len() > MAX_BYTES.saturating_sub(event.len()) { + None + } else { + Some(Arc::from(json)) + } +} + impl PayloadCache { pub(super) fn encode( &self, diff --git a/bt-daemon/src/pipeline/journal.rs b/bt-daemon/src/pipeline/journal.rs index da3dbd1c..ef22a041 100644 --- a/bt-daemon/src/pipeline/journal.rs +++ b/bt-daemon/src/pipeline/journal.rs @@ -191,13 +191,14 @@ impl SpanJournal { std::fs::create_dir_all(parent)?; let db = Connection::open(path)?; db.busy_timeout(std::time::Duration::from_secs(5))?; + db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; BEGIN IMMEDIATE;")?; + // Read the version under the same write transaction as every + // upgrade, so another opener cannot run a stale migration plan. let version: u32 = db.query_row("PRAGMA user_version", [], |row| row.get(0))?; anyhow::ensure!(version <= 7, "unsupported SpanOp journal schema {version}"); - db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;")?; if version < 3 { db.execute_batch( - "BEGIN IMMEDIATE; - CREATE TABLE IF NOT EXISTS batches ( + "CREATE TABLE IF NOT EXISTS batches ( input INTEGER PRIMARY KEY, committed INTEGER NOT NULL DEFAULT 0, observation INTEGER NOT NULL DEFAULT 1); CREATE TABLE IF NOT EXISTS ops ( @@ -216,16 +217,16 @@ impl SpanJournal { )?; } db.execute_batch("ALTER TABLE heads ADD COLUMN binding TEXT")?; - db.execute_batch("PRAGMA user_version=3; COMMIT")?; + db.execute_batch("PRAGMA user_version=3")?; } if version < 4 { - db.execute_batch("BEGIN IMMEDIATE; CREATE TABLE root_candidates (span_id TEXT PRIMARY KEY, value BLOB NOT NULL); PRAGMA user_version=4; COMMIT")?; + db.execute_batch("CREATE TABLE root_candidates (span_id TEXT PRIMARY KEY, value BLOB NOT NULL); PRAGMA user_version=4")?; } if version < 5 { - db.execute_batch("BEGIN IMMEDIATE; ALTER TABLE heads ADD COLUMN settings TEXT; ALTER TABLE heads ADD COLUMN settings_through INTEGER NOT NULL DEFAULT 0; PRAGMA user_version=5; COMMIT")?; + db.execute_batch("ALTER TABLE heads ADD COLUMN settings TEXT; ALTER TABLE heads ADD COLUMN settings_through INTEGER NOT NULL DEFAULT 0; PRAGMA user_version=5")?; } if version < 6 { - db.execute_batch("BEGIN IMMEDIATE; + db.execute_batch(" DELETE FROM ops WHERE input IN (SELECT input FROM batches WHERE committed=0); CREATE TABLE receipts (input INTEGER NOT NULL, ordinal INTEGER NOT NULL, PRIMARY KEY(input,ordinal)); CREATE TABLE publication (id INTEGER PRIMARY KEY CHECK(id=1), tail INTEGER NOT NULL, floor INTEGER NOT NULL); @@ -242,14 +243,15 @@ impl SpanJournal { after = position; } prune(&db)?; - db.execute_batch("PRAGMA user_version=6; COMMIT")?; + db.execute_batch("PRAGMA user_version=6")?; } if version < 7 { - db.execute_batch("BEGIN IMMEDIATE; + db.execute_batch(" CREATE TABLE IF NOT EXISTS translation_checkpoint (id INTEGER PRIMARY KEY CHECK(id=1), input INTEGER NOT NULL, version TEXT NOT NULL, state BLOB NOT NULL); - PRAGMA user_version=7; COMMIT")?; + PRAGMA user_version=7")?; } let tail = db.query_row("SELECT tail FROM publication WHERE id=1", [], |row| row.get(0))?; + db.execute_batch("COMMIT")?; Ok((db, tail)) }) .await??; @@ -835,6 +837,18 @@ impl SpanJournal { }).await } + pub async fn has_head(&self, route: &SessionRoute) -> anyhow::Result { + let route = serde_json::to_string(route)?; + self.with_db(move |db| { + Ok(db.query_row( + "SELECT EXISTS(SELECT 1 FROM heads WHERE route=?)", + [route], + |row| row.get(0), + )?) + }) + .await + } + /// A completed subscription can remain dormant until its source has new work. pub async fn route_complete(&self, route: &SessionRoute, input: u64) -> anyhow::Result { let route = route.clone(); @@ -1621,6 +1635,33 @@ mod tests { assert_eq!(journal.payload_reads.load(Ordering::Relaxed), 3); } + #[tokio::test] + async fn concurrent_opens_upgrade_one_schema_atomically() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("concurrent-upgrade.sqlite"); + let db = Connection::open(&path).unwrap(); + db.execute_batch("PRAGMA journal_mode=WAL; CREATE TABLE batches(input INTEGER PRIMARY KEY,committed INTEGER NOT NULL DEFAULT 0); PRAGMA user_version=1;").unwrap(); + drop(db); + let ready = Arc::new(tokio::sync::Barrier::new(8)); + let mut tasks = Vec::new(); + for _ in 0..8 { + let path = path.clone(); + let ready = ready.clone(); + tasks.push(tokio::spawn(async move { + ready.wait().await; + SpanJournal::open(&path).await + })); + } + let mut opened = Vec::new(); + for task in tasks { + opened.push(task.await.unwrap().unwrap()); + } + for journal in opened { + assert_eq!(journal.tail().await.unwrap(), 0); + assert!(journal.translation_checkpoint().await.unwrap().is_none()); + } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn translation_begin_waits_for_a_concurrent_writer_before_reading() { let tmp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 81675e00..7f8fe04f 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -249,6 +249,10 @@ impl Translation { self.journal.route_settings(route).await } + pub async fn has_head(&self, route: &SessionRoute) -> anyhow::Result { + self.journal.has_head(route).await + } + /// Connect a route to the committed SpanOp stream without exposing its /// storage adapter to the route actor. pub async fn subscribe( @@ -304,6 +308,31 @@ struct Producer { } impl Producer { + /// Translators perform synchronous native transcript I/O and reduction. + /// Keep both replay and live reduction off the runtime serving hook IPC. + async fn translate( + &mut self, + operation: impl FnOnce(&mut dyn AgentTranslator, &SessionCtx) -> anyhow::Result + + Send + + 'static, + ) -> anyhow::Result { + let mut translator = self + .translator + .take() + .ok_or_else(|| anyhow::anyhow!("translator is missing"))?; + let ctx = SessionCtx { + session_id: self.options.session_id.clone(), + config: None, + }; + let (translator, result) = tokio::task::spawn_blocking(move || { + let result = operation(translator.as_mut(), &ctx); + (translator, result) + }) + .await?; + self.translator = Some(translator); + result + } + async fn run(mut self, mut rx: mpsc::Receiver) { let mut recorded = None; let mut record_retry = 0; @@ -462,14 +491,14 @@ impl Producer { .translator .take() .ok_or_else(|| anyhow::anyhow!("translator is missing"))?; - let decoder = self.decoder.clone(); + let decoder = std::mem::take(&mut self.decoder); let capture = self.capture.clone(); let bounds = self.bounds.clone(); let templates = self.templates.clone(); let version = self.options.version.clone(); // Context may include a long active conversation. Its encoding and // compression must not monopolize a runtime thread serving hook IPC. - let (translator, checkpoint) = tokio::task::spawn_blocking(move || { + let (translator, decoder, checkpoint) = tokio::task::spawn_blocking(move || { let checkpoint = (|| -> anyhow::Result> { let Some(state) = translator.save_state()? else { return Ok(None); @@ -479,7 +508,7 @@ impl Producer { translator: serde_json::value::RawValue::from_string(String::from_utf8( state, )?)?, - decoder, + decoder: decoder.clone(), capture, bounds, templates, @@ -493,10 +522,11 @@ impl Producer { state, })) })(); - (translator, checkpoint) + (translator, decoder, checkpoint) }) .await?; self.translator = Some(translator); + self.decoder = decoder; checkpoint } @@ -543,12 +573,15 @@ impl Producer { matches!(self.state, State::Processing), "translation is recovering" ); - self.append_control(kind).await?; + self.initialize().await?; self.advance(self.progress.requested.load(Ordering::Relaxed)) .await?; + if let Some(through) = self.append_control(kind).await? { + self.advance(through).await?; + } self.journal.tail().await } - async fn append_control(&mut self, kind: ControlKind) -> anyhow::Result<()> { + async fn append_control(&mut self, kind: ControlKind) -> anyhow::Result> { self.phase = Boundary::Journal; let mut bounds = HashMap::new(); for path in self.bounds.keys() { @@ -561,6 +594,12 @@ impl Producer { let control = PipelineControl { kind, bounds }; let through = { let mut writer = self.options.events.lock().await; + // A checkpoint observes dispatched work. If capture is ahead, + // flush the published tail and let queued events arrive normally; + // chasing the file here can reorder parent/child processing. + if writer.position() > self.position { + return Ok(None); + } let position = writer.append_control_record(&control).await?; let through = position.through; self.options.input.push(SourceRecord { @@ -572,7 +611,7 @@ impl Producer { self.progress .requested .fetch_max(through, Ordering::Relaxed); - Ok(()) + Ok(Some(through)) } async fn initialize(&mut self) -> anyhow::Result<()> { @@ -589,13 +628,14 @@ impl Producer { } } } + let registry = self.options.registry.clone(); + let source = self.options.source.clone(); + let namespace = self.options.namespace.clone(); self.translator = Some( - self.options - .registry - .create_checked_with_session_namespace( - &self.options.source, - &self.options.namespace, - )?, + tokio::task::spawn_blocking(move || { + registry.create_checked_with_session_namespace(&source, &namespace) + }) + .await??, ); self.position = 0; self.decoder = Default::default(); @@ -626,7 +666,7 @@ impl Producer { &self.options.event_path, input, self.committed, - context.decoder.clone(), + Default::default(), ) .await?; self.translator = Some(translator); @@ -675,7 +715,7 @@ impl Producer { &self.options.event_path, self.position, through, - self.decoder.clone(), + Default::default(), ) .await?, ); @@ -685,10 +725,12 @@ impl Producer { .as_mut() .ok_or_else(|| anyhow::anyhow!("recovery reader is missing"))?; reader.extend(through); - let Some(entry) = reader.next_record().await? else { + reader.swap_decoder_context(&mut self.decoder); + let entry = reader.next_record().await; + reader.swap_decoder_context(&mut self.decoder); + let Some(entry) = entry? else { return Ok(None); }; - self.decoder = reader.decoder_context().clone(); let event = match entry.record { JournalRecord::Event(event) => { SourceEvent::Event(Box::new(crate::journal::envelope_from_redacted(event))) @@ -725,10 +767,6 @@ impl Producer { while let Some(entry) = self.next_input(through).await? { let input = entry.position.through; let observation = !matches!(&entry.event,SourceEvent::Control(control) if matches!(control.kind,ControlKind::Retire)); - let ctx = SessionCtx { - session_id: self.options.session_id.clone(), - config: None, - }; let (first, capture) = match entry.event { SourceEvent::Event(mut env) => { if let Some(route) = &env.route { @@ -791,10 +829,8 @@ impl Producer { self.capture = Some(capture.clone()); self.phase = Boundary::Translation; ( - self.translator - .as_mut() - .ok_or_else(|| anyhow::anyhow!("translator is missing"))? - .handle(&env, &ctx)?, + self.translate(move |translator, ctx| translator.handle(&env, ctx)) + .await?, Some(capture), ) } @@ -812,15 +848,15 @@ impl Producer { self.checkpoint_safe = false; } self.phase = Boundary::Translation; - let translator = self - .translator - .as_mut() - .ok_or_else(|| anyhow::anyhow!("translator is missing"))?; - let ops = if !matches!(control.kind, ControlKind::Checkpoint) { - translator.finalize_at(&ctx, &control.bounds)? - } else { - translator.checkpoint_at(&ctx, &control.bounds)? - }; + let ops = self + .translate(move |translator, ctx| { + if !matches!(control.kind, ControlKind::Checkpoint) { + translator.finalize_at(ctx, &control.bounds) + } else { + translator.checkpoint_at(ctx, &control.bounds) + } + }) + .await?; (ops, self.capture.clone()) } SourceEvent::LegacyDeliveryCheckpoint => { @@ -861,10 +897,6 @@ impl Producer { .ok_or_else(|| anyhow::anyhow!("pending publication is missing"))?; let result = async { let publish = pending.input > self.committed; - let ctx = SessionCtx { - session_id: self.options.session_id.clone(), - config: None, - }; if publish && !pending.begun { self.phase = Boundary::Journal; self.journal @@ -935,10 +967,8 @@ impl Producer { } self.phase = Boundary::Translation; pending.ops = self - .translator - .as_mut() - .ok_or_else(|| anyhow::anyhow!("translator is missing"))? - .drain_pending(&ctx)?; + .translate(|translator, ctx| translator.drain_pending(ctx)) + .await?; pending.complete = pending.ops.is_none(); pending.prepared = false; } @@ -1049,6 +1079,96 @@ impl Producer { mod tests { use super::*; use crate::{pipeline::delivery::InputPort, translate::SpanRow}; + mod streams { + use crate as bt_daemon; + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/translation_streams.rs" + )); + } + + #[tokio::test] + async fn native_source_output_matches_live_overflow_and_cold_recovery() { + fn comparable(ops: Vec<(u64, SpanOp)>) -> Vec { + let mut rows = ops + .into_iter() + .map(|(_, op)| { + let (SpanOp::Insert(row) | SpanOp::Merge(row)) = &op; + serde_json::to_string(&( + &op, + &row.turn_span_id, + row.is_turn, + &row.late_merge_key, + &row.late_merge_version, + )) + .unwrap() + }) + .collect::>(); + rows.sort(); + rows + } + for (source, mut events, _fixtures) in streams::native_streams() { + events.insert(0, events[0].clone()); + for event in &mut events { + event.payload["codec_padding"] = serde_json::json!("é📚history".repeat(1024)); + } + let mut expected = None; + for mode in ["live", "overflow", "cold"] { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), &source, "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let options = resume_options(tmp.path(), &source, writer.clone()); + let input = options.input.clone(); + let mut translation = Translation::spawn(options).await.unwrap(); + let mut positions = Vec::new(); + for (index, event) in events.iter().enumerate() { + let through = if mode == "cold" && index > 0 { + writer.lock().await.append(event).await.unwrap() + } else { + capture(&writer, &input, event).await + }; + positions.push(through); + } + if mode == "overflow" { + // Emulate losing the first two queued records while their + // complete persisted forms remain recoverable. + assert!(input.take(0, positions[0]).is_some()); + assert!(input.take(positions[0], positions[1]).is_some()); + } + if mode == "cold" { + translation.advance(positions[0]).await.unwrap(); + translation.barrier().await.unwrap(); + translation.shutdown().await; + // New writer and input: the following stored delta must + // rely on the SQLite decoder/reducer checkpoint alone. + *writer.lock().await = JournalWriter::open_path(&path).await.unwrap(); + translation = + Translation::spawn(resume_options(tmp.path(), &source, writer.clone())) + .await + .unwrap(); + } + translation + .advance(*positions.last().unwrap()) + .await + .unwrap(); + translation + .barrier() + .await + .unwrap_or_else(|error| panic!("{source} {mode}: {error}")); + translation.control(ControlKind::Finalize).await.unwrap(); + let rows = comparable(translation.journal.read_after(0, 10000).await.unwrap()); + assert!(!rows.is_empty(), "{source} produced no observations"); + if let Some(expected) = &expected { + assert_eq!(&rows, expected, "{source} {mode}"); + } else { + expected = Some(rows); + } + translation.shutdown().await; + } + } + } fn resume_options( data_dir: &std::path::Path, @@ -1070,6 +1190,38 @@ mod tests { } } + #[tokio::test] + async fn checkpoints_cannot_absorb_captured_events_before_dispatch() { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let options = resume_options(tmp.path(), "debug", writer.clone()); + let input = options.input.clone(); + let source = Translation::spawn(options).await.unwrap(); + let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{"index":0}})).unwrap(); + let first = capture(&writer, &input, &event).await; + source.advance(first).await.unwrap(); + source.barrier().await.unwrap(); + event.payload["index"] = 1.into(); + let second = capture(&writer, &input, &event).await; + // Capture has acknowledged a later event whose routing/ancestry has + // not been dispatched. A flush must not translate it ahead of that. + source.control(ControlKind::Checkpoint).await.unwrap(); + assert_eq!(source.journal.translated_through().await.unwrap(), first); + assert_eq!( + writer.lock().await.position(), + second, + "no control marker should leap over undispatched work" + ); + source.advance(second).await.unwrap(); + source.barrier().await.unwrap(); + source.control(ControlKind::Checkpoint).await.unwrap(); + assert!(source.journal.translated_through().await.unwrap() > second); + source.shutdown().await; + } + #[tokio::test] async fn live_translation_and_controls_do_not_read_events_back_from_disk() { let tmp = tempfile::tempdir().unwrap(); @@ -1203,6 +1355,103 @@ mod tests { entered: Arc, release: std::sync::mpsc::Receiver<()>, } + + struct BlockedReplay { + entered: Arc, + finished: Arc, + release: std::sync::mpsc::Receiver<()>, + } + impl AgentTranslator for BlockedReplay { + fn handle( + &mut self, + _: &crate::wire::Envelope, + _: &SessionCtx, + ) -> anyhow::Result> { + self.entered.notify_one(); + let _ = self.release.recv_timeout(std::time::Duration::from_secs(5)); + self.finished.store(true, Ordering::Release); + Ok(Vec::new()) + } + } + + #[tokio::test] + async fn synchronous_replay_work_cannot_block_the_hook_runtime() { + let tmp = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{}})).unwrap(); + let through = writer.lock().await.append(&event).await.unwrap(); + let options = resume_options(tmp.path(), "debug", writer.clone()); + let input = options.input.clone(); + let entered = Arc::new(Notify::new()); + let finished = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (release, receiver) = std::sync::mpsc::channel(); + let mut producer = Producer { + options, + journal: SpanJournal::open(&tmp.path().join("work.sqlite")) + .await + .unwrap(), + progress: Arc::default(), + error: Arc::default(), + recovery: Arc::default(), + translator: Some(Box::new(BlockedReplay { + entered: entered.clone(), + finished: finished.clone(), + release: receiver, + })), + reader: None, + decoder: Default::default(), + observers: Vec::new(), + origins: WeakOrigins::default(), + roots: Roots::default(), + capture: None, + pending_snapshots: HashSet::new(), + bounds: HashMap::new(), + templates: HashMap::new(), + position: 0, + committed: 0, + checkpoint_position: 0, + checkpoint_events: 0, + checkpoint_work_bytes: 0, + checkpoint_safe: true, + state: State::Processing, + publication: None, + phase: Boundary::Journal, + }; + let started = std::time::Instant::now(); + let processing = tokio::spawn(async move { + let result = producer.advance(through).await; + (producer, result) + }); + entered.notified().await; + let elapsed = started.elapsed(); + event.ts_ms = 2; + let captured = tokio::time::timeout( + std::time::Duration::from_secs(1), + capture(&writer, &input, &event), + ) + .await; + let replay_still_blocked = !finished.load(Ordering::Acquire); + let _ = release.send(()); + let (producer, result) = processing.await.unwrap(); + assert!( + elapsed < std::time::Duration::from_secs(1), + "replay monopolized the hook runtime for {elapsed:?}" + ); + assert!(captured.unwrap() > through); + assert!( + replay_still_blocked, + "capture only completed after synchronous replay returned" + ); + result.unwrap(); + assert_eq!( + producer.journal.translated_through().await.unwrap(), + through, + "replay must retain its original stopping point" + ); + } impl AgentTranslator for BlockedCheckpoint { fn handle( &mut self, diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 041177e3..77f0f8d2 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -25,7 +25,7 @@ impl StartupBuffer { } } - fn push(&mut self, data_dir: &std::path::Path, event: PendingEvent) -> anyhow::Result<()> { + fn push(&mut self, data_dir: &std::path::Path, event: PendingEvent) { if self.events.len() == INGRESS_QUEUE_CAPACITY { if let Some(oldest) = self.events.pop_front() { let path = journal::source_journal_path( @@ -43,7 +43,6 @@ impl StartupBuffer { } } self.events.push_back(event); - Ok(()) } } @@ -54,26 +53,26 @@ pub(super) enum StartupWork { } impl Daemon { - pub(super) async fn next_startup_work(&self) -> anyhow::Result { + pub(super) fn next_startup_work(&self) -> StartupWork { let mut state = self.startup_buffer.lock(); let Some(buffer) = state.as_mut() else { - return Ok(StartupWork::Finished); + return StartupWork::Finished; }; if !buffer.evicted.is_empty() { - return Ok(StartupWork::Evicted(std::mem::take(&mut buffer.evicted))); + return StartupWork::Evicted(std::mem::take(&mut buffer.evicted)); } if let Some(event) = buffer.events.pop_front() { - return Ok(StartupWork::Event(Box::new(event))); + return StartupWork::Event(Box::new(event)); } *state = None; - Ok(StartupWork::Finished) + StartupWork::Finished } } pub(super) async fn drain_startup_buffer(daemon: &Arc) -> anyhow::Result<()> { let mut cursors = HashMap::new(); loop { - match daemon.next_startup_work().await? { + match daemon.next_startup_work() { StartupWork::Evicted(bounds) => { replay_evicted_startup(daemon, &mut cursors, &bounds).await? } @@ -83,7 +82,7 @@ pub(super) async fn drain_startup_buffer(daemon: &Arc) -> anyhow::Result } } -pub(super) struct StartupCursor { +pub(super) struct ReplayCursor { reader: journal::JournalReader, position: u64, } @@ -93,7 +92,7 @@ pub(super) struct StartupCursor { /// only to initialize that context; they are not dispatched again. pub(super) async fn replay_evicted_startup( daemon: &Arc, - cursors: &mut HashMap, + cursors: &mut HashMap, bounds: &HashMap, ) -> anyhow::Result<()> { for (path, range) in bounds { @@ -111,7 +110,7 @@ pub(super) async fn replay_evicted_startup( 0, ), }; - cursors.insert(path.clone(), StartupCursor { reader, position }); + cursors.insert(path.clone(), ReplayCursor { reader, position }); } let cursor = cursors .get_mut(path) @@ -168,17 +167,30 @@ pub(super) async fn replay_evicted_startup( async fn next_evicted_event( daemon: &Daemon, - cursor: &mut StartupCursor, + cursor: &mut ReplayCursor, through: u64, ) -> anyhow::Result> { + let Some((sequence, event)) = next_replayed_event(daemon, cursor, through).await? else { + return Ok(None); + }; + let sequence = + sequence.ok_or_else(|| anyhow::anyhow!("live startup event has no capture sequence"))?; + Ok(Some((sequence, event))) +} + +async fn next_replayed_event( + daemon: &Daemon, + cursor: &mut ReplayCursor, + through: u64, +) -> anyhow::Result, PendingEvent)>> { while cursor.position < through { let before = cursor.position; let entry = cursor .reader .next_record() .await? - .ok_or_else(|| anyhow::anyhow!("evicted startup range ends inside a record"))?; - anyhow::ensure!(entry.through <= through, "invalid evicted startup end"); + .ok_or_else(|| anyhow::anyhow!("replay range ends inside a record"))?; + anyhow::ensure!(entry.through <= through, "invalid replay end boundary"); cursor.position = entry.through; if let journal::JournalRecord::Event(redacted) = entry.record { let mut env = journal::envelope_from_redacted(redacted); @@ -186,11 +198,8 @@ async fn next_evicted_event( continue; }; env.source = source.to_owned(); - let sequence = entry - .capture_sequence - .ok_or_else(|| anyhow::anyhow!("live startup event has no capture sequence"))?; return Ok(Some(( - sequence, + entry.capture_sequence, PendingEvent { env, replay_through: before, @@ -200,7 +209,7 @@ async fn next_evicted_event( ))); } } - anyhow::ensure!(cursor.position == through, "invalid evicted startup end"); + anyhow::ensure!(cursor.position == through, "invalid replay end boundary"); Ok(None) } @@ -314,102 +323,147 @@ pub(super) async fn dispatch_ingress_event(daemon: &Arc, event: PendingE } } -/// Drain events omitted from the bounded in-memory queue. The queue contains -/// only a latency fast path; journals remain the complete source of ingress. +struct OverflowCursor { + path: PathBuf, + replay: ReplayCursor, + through: u64, + acknowledged: HashMap, +} + +type OverflowEvent = ((bool, u64), PendingEvent); + +async fn next_overflow_event( + daemon: &Arc, + cursor: &mut OverflowCursor, +) -> Option { + loop { + let (sequence, event) = match next_replayed_event( + daemon, + &mut cursor.replay, + cursor.through, + ) + .await + { + Ok(Some(event)) => event, + Ok(None) => return None, + Err(error) => { + tracing::warn!(%error, path=%cursor.path.display(), "ingress overflow replay failed"); + return None; + } + }; + if event.journal_through + <= daemon + .ingress_history + .lock() + .get(&cursor.path) + .copied() + .unwrap_or(0) + { + continue; + } + let Some(route) = event.env.route.as_ref() else { + continue; + }; + let Ok(key) = DeliveryKey::new(&event.env.source, &event.env.session_id, route) else { + continue; + }; + let dispatched = daemon + .ingress_dispatched + .lock() + .get(&key) + .copied() + .unwrap_or(0); + if event.journal_through <= dispatched { + continue; + } + let route_json = recovered_delivery_route(daemon, &event.env) + .await + .as_ref() + .and_then(|route| serde_json::to_string(route).ok()) + .unwrap_or_else(|| serde_json::to_string(route).unwrap_or_default()); + if event.journal_through <= cursor.acknowledged.get(&route_json).copied().unwrap_or(0) { + continue; + } + return Some(((sequence.is_some(), sequence.unwrap_or(0)), event)); + } +} + +/// Recover a frozen durable range, merging one lookahead per source by capture +/// ordinal. Per-file replay would close a parent's tool before dispatching a +/// child captured between that tool's start and completion. pub(super) async fn recover_ingress_overflow(daemon: &Arc) { let Ok(mut entries) = tokio::fs::read_dir(journal::journal_dir(&daemon.data_dir)).await else { return; }; - let mut recovered = 0usize; + let mut ready = Vec::new(); while let Ok(Some(entry)) = entries.next_entry().await { let path = entry.path(); if path.extension().and_then(|ext| ext.to_str()) != Some("ndjson") { continue; } - let recorded_len = journal::JournalReader::recorded_len(&path).await; - let Ok(Some(mut checkpoints)) = journal::JournalReader::open(&path, recorded_len).await - else { - continue; - }; - let mut start = 0; - if let Ok(Some((reader, position, _))) = resume_discovery(daemon, &path, recorded_len).await + let through = daemon.journal_snapshot(&path).await; + if through + <= daemon + .ingress_history + .lock() + .get(&path) + .copied() + .unwrap_or(0) { - checkpoints = reader; - start = position; - } - let decoder = checkpoints.decoder_context().clone(); - let mut acknowledged_by_route: HashMap = HashMap::new(); - while let Ok(Some(entry)) = checkpoints.next_record().await { - if let journal::JournalRecord::DeliveryCheckpoint { route, through } = entry.record { - let key = serde_json::to_string(&route).unwrap_or_default(); - let acknowledged = acknowledged_by_route.entry(key).or_default(); - *acknowledged = (*acknowledged).max(through); - } + continue; } - - let mut reader = if start > 0 { - let Ok(reader) = - journal::JournalReader::resume(&path, start, recorded_len, decoder).await - else { - continue; - }; - reader + let mut acknowledged = HashMap::::new(); + let (reader, start) = if let Ok(Some((reader, position, _))) = + resume_discovery(daemon, &path, through).await + { + // A compatible checkpoint already uses SQLite delivery heads. + (reader, position) } else { - let Ok(Some(reader)) = journal::JournalReader::open(&path, recorded_len).await else { - continue; - }; - reader - }; - let mut before = start; - while let Ok(Some(entry)) = reader.next_record().await { - let through = entry.through; - let journal::JournalRecord::Event(redacted) = entry.record else { - before = through; - continue; - }; - let mut env = journal::envelope_from_redacted(redacted); - let Some(source) = daemon.translators.canonical_source(&env.source) else { - before = through; - continue; - }; - env.source = source.to_string(); - let Some(route) = env.route.as_ref() else { - before = through; + let Ok(Some(mut legacy)) = journal::JournalReader::open(&path, through).await else { continue; }; - let route_json = recovered_delivery_route(daemon, &env) - .await - .as_ref() - .and_then(|route| serde_json::to_string(route).ok()) - .unwrap_or_else(|| serde_json::to_string(route).unwrap_or_default()); - let key = match DeliveryKey::new(&env.source, &env.session_id, route) { - Ok(key) => key, - Err(_) => { - before = through; - continue; + while let Ok(Some(entry)) = legacy.next_record().await { + if let journal::JournalRecord::DeliveryCheckpoint { route, through } = entry.record + { + let key = serde_json::to_string(&route).unwrap_or_default(); + let head = acknowledged.entry(key).or_default(); + *head = (*head).max(through); } - }; - let dispatched = daemon - .ingress_dispatched - .lock() - .get(&key) - .copied() - .unwrap_or(0); - let acknowledged = acknowledged_by_route.get(&route_json).copied().unwrap_or(0); - if through > dispatched.max(acknowledged) { - dispatch_ingress_event( - daemon, - PendingEvent { - env, - replay_through: before, - journal_through: through, - settings_through: None, - }, - ) - .await; - recovered += 1; } - before = through; + let Ok(Some(reader)) = journal::JournalReader::open(&path, through).await else { + continue; + }; + (reader, 0) + }; + let mut cursor = OverflowCursor { + path, + replay: ReplayCursor { + reader, + position: start, + }, + through, + acknowledged, + }; + if let Some((order, event)) = next_overflow_event(daemon, &mut cursor).await { + ready.push((order, event, cursor)); + } + } + let mut recovered = 0; + while !ready.is_empty() { + let Some(index) = ready + .iter() + .enumerate() + .min_by_key(|(_, (order, _, _))| *order) + .map(|(index, _)| index) + else { + break; + }; + let (_, event, mut cursor) = ready.swap_remove(index); + debug_assert!(event.journal_through <= cursor.through); + dispatch_ingress_event(daemon, event).await; + recovered += 1; + if let Some((order, event)) = next_overflow_event(daemon, &mut cursor).await { + ready.push((order, event, cursor)); } } if recovered > 0 { @@ -578,10 +632,7 @@ impl Daemon { let event = { let mut state = self.startup_buffer.lock(); if let Some(buffer) = state.as_mut() { - if let Err(error) = buffer.push(&self.data_dir, event) { - self.trigger_shutdown(); - return Err(format!("startup index failed: {error}")); - } + buffer.push(&self.data_dir, event); return Ok(true); } event diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index abccb6f4..55f92f75 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -84,7 +84,7 @@ async fn collect_partitions(daemon: &Daemon, age: Duration) { let lock = daemon.session_lock(&first.source, &first.session_id); let _guard = lock.lock().await; let namespace = crate::ids::session_namespace(&first.source, &first.session_id); - if daemon.journals.lock().contains_key(&namespace) + if daemon.journals.lock().contains_key(&path) || daemon.translations.lock().contains_key(&namespace) { continue; @@ -131,6 +131,11 @@ async fn collect_partitions(daemon: &Daemon, age: Duration) { if tokio::fs::remove_file(&storage).await.is_err() { continue; } + daemon.ingress_history.lock().remove(&path); + daemon + .ingress_dispatched + .lock() + .retain(|key, _| key.source != first.source || key.session_id != first.session_id); for sidecar in [wal, storage.with_extension("sqlite-shm"), path] { if let Err(error) = tokio::fs::remove_file(&sidecar).await { if error.kind() != std::io::ErrorKind::NotFound { @@ -663,7 +668,7 @@ mod tests { next.payload = serde_json::json!({"index": i}); assert!(daemon.capture_event(next).await.unwrap()); } - let StartupWork::Evicted(first) = daemon.next_startup_work().await.unwrap() else { + let StartupWork::Evicted(first) = daemon.next_startup_work() else { panic!("the oldest events were not evicted"); }; for i in INGRESS_QUEUE_CAPACITY + 2..INGRESS_QUEUE_CAPACITY + 7 { @@ -682,7 +687,7 @@ mod tests { replay_evicted_startup(&daemon, &mut cursors, &first) .await .unwrap(); - let StartupWork::Evicted(second) = daemon.next_startup_work().await.unwrap() else { + let StartupWork::Evicted(second) = daemon.next_startup_work() else { panic!("new evictions did not advance recovery"); }; replay_evicted_startup(&daemon, &mut cursors, &second) @@ -710,6 +715,48 @@ mod tests { drain_all(&daemon).await; } + #[tokio::test] + async fn overflow_snapshot_excludes_unacknowledged_writes() { + let tmp = tempfile::tempdir().unwrap(); + let (ready, startup) = oneshot::channel(); + let mut opts = debug_serve_options("test", tmp.path()); + opts.auth_provider = Some(Arc::new(TestAuth)); + let daemon = Daemon::new_with_startup(opts, tmp.path().into(), Some(startup)); + *daemon.startup_buffer.lock() = None; + let mut env = event("durable-bound", SessionRoute::default()); + env.payload = serde_json::json!({"index":0}); + daemon.capture_event(env.clone()).await.unwrap(); + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + let acknowledged = tokio::fs::read(&path).await.unwrap(); + let through = acknowledged.len() as u64; + // OS-visible bytes need not have completed fsync. Model a complete + // record written after the writer's acknowledged durable position. + env.payload = serde_json::json!({"index":1}); + let mut unacknowledged = acknowledged.clone(); + unacknowledged.extend(serde_json::to_vec(&env.redacted()).unwrap()); + unacknowledged.push(b'\n'); + tokio::fs::write(&path, unacknowledged).await.unwrap(); + recover_ingress_overflow(&daemon).await; + let source = daemon.translations.lock().values().next().unwrap().clone(); + source.barrier().await.unwrap(); + let storage = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&env.source, &env.session_id) + )); + let partition = crate::pipeline::journal::SpanJournal::open(&storage) + .await + .unwrap(); + assert_eq!( + partition.translated_through().await.unwrap(), + through, + "recovery must not commit beyond the durable capture head" + ); + tokio::fs::write(&path, acknowledged).await.unwrap(); + ready.send(()).unwrap(); + daemon.settle_ingress().await; + drain_all(&daemon).await; + } + #[tokio::test] async fn startup_overflow_preserves_parent_child_tool_order() { use distributed_fixture::{AgentKind, DistributedFixtures, ProcessTree}; @@ -773,6 +820,72 @@ mod tests { drain_all(&daemon).await; } + #[tokio::test] + async fn steady_overflow_preserves_parent_child_tool_order() { + use distributed_fixture::{AgentKind, DistributedFixtures, ProcessTree}; + let tmp = tempfile::tempdir().unwrap(); + let (ready, startup) = oneshot::channel(); + let mut opts = debug_serve_options("test", tmp.path()); + opts.auth_provider = Some(Arc::new(TestAuth)); + let daemon = Daemon::new_with_startup(opts, tmp.path().into(), Some(startup)); + // Pause the steady ingress worker while captures saturate its queue. + *daemon.startup_buffer.lock() = None; + let mut fixtures = DistributedFixtures::new(); + let parent = ProcessTree::root(1000); + let child = ProcessTree::child(1002, 1001, &parent); + let mut events = + fixtures.start_turn(AgentKind::OpenCode, "parent", &parent, "delegate", 100); + events.push(fixtures.open_tool( + AgentKind::OpenCode, + "parent", + &parent, + "tool", + "do work", + 102, + )); + let child_events = + fixtures.start_turn(AgentKind::OpenCode, "child", &child, "do work", 103); + let child_key = automatic_link_key(&child_events[0]); + events.extend(child_events); + events.push(fixtures.close_tool( + AgentKind::OpenCode, + "parent", + &parent, + "tool", + "do work", + "done", + 105, + )); + for env in events { + daemon.capture_event(env).await.unwrap(); + } + for i in 0..INGRESS_QUEUE_CAPACITY { + let mut filler = event("filler", SessionRoute::default()); + filler.ts_ms = i as i64; + daemon.capture_event(filler).await.unwrap(); + } + assert!(daemon.ingress_overflow.load(Ordering::Acquire)); + recover_ingress_overflow(&daemon).await; + ready.send(()).unwrap(); + daemon.settle_ingress().await; + let route = daemon + .automatic_links + .lock() + .get(&child_key) + .cloned() + .expect("child must be linked"); + let Some(crate::wire::TraceDestination::ParentSpan { components }) = route.destination + else { + panic!("child was not attached to the parent tool"); + }; + let tool_id = crate::ids::span_id( + &crate::ids::session_namespace("opencode", "parent"), + "tool:parent:tool", + ); + assert_eq!(components.span_id.as_deref(), Some(tool_id.as_str())); + drain_all(&daemon).await; + } + #[tokio::test] async fn startup_overflow_tracks_each_source_journal_independently() { let tmp = tempfile::tempdir().unwrap(); @@ -836,7 +949,7 @@ mod tests { env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); assert!(daemon.capture_event(env.clone()).await.unwrap()); } - let StartupWork::Evicted(first) = daemon.next_startup_work().await.unwrap() else { + let StartupWork::Evicted(first) = daemon.next_startup_work() else { panic!("missing first overflow") }; let mut cursors = HashMap::new(); @@ -848,7 +961,7 @@ mod tests { env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); assert!(daemon.capture_event(env.clone()).await.unwrap()); } - let StartupWork::Evicted(second) = daemon.next_startup_work().await.unwrap() else { + let StartupWork::Evicted(second) = daemon.next_startup_work() else { panic!("missing second overflow") }; replay_evicted_startup(&daemon, &mut cursors, &second) diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index 949fba45..e5a32ab8 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -102,6 +102,8 @@ struct SessionAuthState { lease: AuthLease, } +type JournalSlot = Arc>>>; + pub struct Daemon { version: String, data_dir: PathBuf, @@ -115,7 +117,7 @@ pub struct Daemon { /// Serializes daemon-side routing and actor creation independently of /// capture, which must never inherit actor or sink backpressure. dispatch_locks: Mutex>>>, - journals: Mutex>>>, + journals: Mutex>, source_inputs: Mutex>, claude_transcripts: Mutex>, managed_run_sessions: Mutex>>, @@ -132,6 +134,9 @@ pub struct Daemon { ingress_tx: mpsc::Sender, ingress_overflow: AtomicBool, ingress_dispatched: Mutex>, + /// Historical prefixes already considered by startup recovery. Capture + /// ordinals in those prefixes belong to an earlier daemon incarnation. + ingress_history: Mutex>, /// Earliest live append during startup, keyed by source journal path. /// Recovery must not absorb queued live events into its historical head. startup_cutoffs: Mutex>>, @@ -189,6 +194,7 @@ impl Daemon { ingress_tx, ingress_overflow: AtomicBool::new(false), ingress_dispatched: Mutex::new(HashMap::new()), + ingress_history: Mutex::new(HashMap::new()), startup_cutoffs: Mutex::new(startup_cutoffs), startup_buffer: Mutex::new(startup_buffer), capture_sequence: AtomicU64::new(0), @@ -277,7 +283,9 @@ pub async fn run(args: ServeArgs, opts: ServeOptions) -> anyhow::Result<()> { .await; restore_pending_sessions(&recovery_daemon).await; let candidates = discover_unprocessed_journals(&recovery_daemon).await; - *recovery_daemon.startup_cutoffs.lock() = None; + if let Some(history) = recovery_daemon.startup_cutoffs.lock().take() { + recovery_daemon.ingress_history.lock().extend(history); + } for event in candidates { dispatch_ingress_event(&recovery_daemon, event).await; } diff --git a/bt-daemon/src/server/recovery.rs b/bt-daemon/src/server/recovery.rs index 738f30ea..75e669b3 100644 --- a/bt-daemon/src/server/recovery.rs +++ b/bt-daemon/src/server/recovery.rs @@ -81,13 +81,16 @@ pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec

anyhow::Result>> { - let storage_key = crate::ids::session_namespace(source, session_id); - if let Some(writer) = self.journals.lock().get(&storage_key).cloned() { - return Ok(writer); - } - let path = journal::ensure_source_journal(&self.data_dir, source, session_id).await?; - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await?, - )); - Ok(self + let path = journal::source_journal_path(&self.data_dir, source, session_id); + let slot = self .journals .lock() - .entry(storage_key) - .or_insert_with(|| writer.clone()) - .clone()) + .entry(path.clone()) + .or_default() + .clone(); + // A recovery actor and capture may open the same source concurrently. + // Tail repair must finish exactly once before either can append. + let writer = slot + .get_or_try_init(|| async { + let path = + journal::ensure_source_journal(&self.data_dir, source, session_id).await?; + let writer = JournalWriter::open_path(&path).await?; + let mut history = self.ingress_history.lock(); + if history + .get(&path) + .is_some_and(|before| *before > writer.position()) + { + history.remove(&path); + } + Ok::<_, anyhow::Error>(Arc::new(tokio::sync::Mutex::new(writer))) + }) + .await?; + Ok(writer.clone()) + } + + /// Take only the durable append position, never hold the writer while + /// reading/replaying records. Recheck after metadata I/O if it opened meanwhile. + pub(super) async fn journal_snapshot(&self, path: &std::path::Path) -> u64 { + let slot = self.journals.lock().get(path).cloned(); + if let Some(writer) = slot.and_then(|slot| slot.get().cloned()) { + return writer.lock().await.position(); + } + let recorded = journal::JournalReader::recorded_len(path).await; + let slot = self.journals.lock().get(path).cloned(); + if let Some(writer) = slot.and_then(|slot| slot.get().cloned()) { + recorded.min(writer.lock().await.position()) + } else { + recorded + } } pub(super) fn source_input_for( diff --git a/bt-daemon/tests/support/translation_streams.rs b/bt-daemon/tests/support/translation_streams.rs new file mode 100644 index 00000000..00057179 --- /dev/null +++ b/bt-daemon/tests/support/translation_streams.rs @@ -0,0 +1,159 @@ +// Native event streams shared by reducer and storage-equivalence tests. +use bt_daemon as fixture_sdk; +#[allow(dead_code)] +mod distributed { + use super::fixture_sdk as bt_daemon; + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/distributed.rs" + )); +} +use bt_daemon::wire::Envelope; +use distributed::{AgentKind, DistributedFixtures, ProcessTree}; +use serde_json::{json, Value}; +fn event(source: &str, name: &str, ts: i64, payload: Value) -> Envelope { + Envelope { + source: source.into(), + source_version: None, + plugin_version: None, + session_id: "checkpoint".into(), + event: name.into(), + ts_ms: ts, + managed_run_id: None, + capture: None, + payload, + route: None, + config: None, + } +} +// Keep the fixtures alive while consuming a stream: Codex events refer to +// transcript files owned by this guard. +pub type NativeStream = (String, Vec, Option); + +pub fn native_streams() -> Vec { + let mut cases = Vec::::new(); + { + for kind in AgentKind::ALL { + let mut fixtures = DistributedFixtures::new(); + let tree = ProcessTree::root(1234); + let mut events = + fixtures.start_turn(kind, "checkpoint", &tree, "first", 1_700_000_000_000); + events.push(fixtures.open_tool( + kind, + "checkpoint", + &tree, + "call", + "work", + 1_700_000_000_010, + )); + events.push(fixtures.close_tool( + kind, + "checkpoint", + &tree, + "call", + "work", + "result", + 1_700_000_000_020, + )); + events.push(fixtures.close_session( + kind, + "checkpoint", + &tree, + "complete", + 1_700_000_000_030, + )); + events.extend(fixtures.start_turn( + kind, + "checkpoint", + &tree, + "second", + 1_700_000_000_040, + )); + let source = events[0].source.clone(); + cases.push((source.clone(), events, Some(fixtures))); + } + } + { + let events = include_str!("../fixtures/muse-hooks-1.1.1.jsonl") + .lines() + .enumerate() + .map(|(i, line)| { + let payload: Value = serde_json::from_str(line).unwrap(); + let name = payload["hook_event_name"].as_str().unwrap().to_owned(); + event("muse", &name, i as i64 + 1, payload) + }) + .collect(); + cases.push(("muse".to_owned(), events, None)); + } + { + let events = [ + ("sessionStart", json!({})), + ("beforeSubmitPrompt", json!({"prompt":"work"})), + ( + "preToolUse", + json!({"tool_name":"Shell","tool_use_id":"call","tool_input":{"command":"echo hello"}}), + ), + ( + "postToolUse", + json!({"tool_name":"Shell","tool_use_id":"call","tool_output":"hello","tool_input":{"command":"echo hello"},"duration":1}), + ), + ("afterAgentResponse", json!({"text":"done"})), + ("stop", json!({"status":"completed"})), + ("beforeSubmitPrompt", json!({"prompt":"next"})), + ] + .into_iter() + .enumerate() + .map(|(i, (name, mut payload))| { + payload["hook_event_name"] = name.into(); + payload["conversation_id"] = "checkpoint".into(); + payload["generation_id"] = format!("generation-{i}").into(); + payload["model"] = "test".into(); + payload["workspace_roots"] = json!(["/workspace"]); + event("cursor", name, i as i64 + 1, payload) + }) + .collect(); + cases.push(("cursor".to_owned(), events, None)); + } + { + let records = [ + json!({"step_index":0,"source":"USER_EXPLICIT","type":"USER_INPUT","status":"COMPLETED","content":"List files"}), + json!({"step_index":1,"source":"MODEL","type":"PLANNER_RESPONSE","status":"COMPLETED","content":"Inspecting","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"}}],"usage":{"inputTokens":12,"outputTokens":7,"totalTokens":19}}), + json!({"step_index":2,"source":"MODEL","type":"CORTEX_STEP_TYPE_RUN_COMMAND","status":"COMPLETED","content":"README","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"},"result":"README"}]}), + ]; + let mut contents = String::new(); + let mut boundaries = Vec::new(); + for record in records { + contents.push_str(&serde_json::to_string(&record).unwrap()); + contents.push('\n'); + boundaries.push(contents.len()); + } + let events = [("PreInvocation", boundaries[0]), ("PostInvocation", boundaries[1]), ("PostToolUse", boundaries[2]), ("Stop", boundaries[2])].into_iter().enumerate().map(|(i, (name, through))| { + event("antigravity", name, i as i64 + 1, json!({"conversationId":"checkpoint","invocationNum":0,"initialNumSteps":1,"stepIdx":2,"fullyIdle":true, + "transcriptPath":"/tmp/checkpoint/transcript.jsonl", + "_bt_transcript_observation":{"path":"/tmp/checkpoint/transcript.jsonl","observed_bytes":through,"full_path":"/tmp/checkpoint/transcript_full.jsonl","full_observed_bytes":through}, + "_bt_transcript_snapshot":{"path":"/tmp/checkpoint/transcript_full.jsonl","contents":contents}})) + }).collect(); + cases.push(("antigravity".to_owned(), events, None)); + } + { + let root = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/grok/transcript"); + let updates = root.join("updates.jsonl"); + let events = root.join("events.jsonl"); + let mut offset = 0; + let stream = std::fs::read(&updates).unwrap().split_inclusive(|byte| *byte == b'\n').map(|line| { + offset += line.len() as u64; + event("grok", "Wake", 1600, json!({"cwd":"/repo","_bt_grok_transcript_mirrors":{"updates":{"mirror":updates,"through":offset},"events":{"mirror":events,"through":std::fs::metadata(&events).unwrap().len()}}})) + }).collect(); + cases.push(("grok".to_owned(), stream, None)); + } + cases.push(( + "debug".into(), + vec![ + event("debug", "event", 1, json!({"value":1})), + event("debug", "event", 2, json!({"value":2})), + ], + None, + )); + cases +} diff --git a/bt-daemon/tests/translator_checkpoints.rs b/bt-daemon/tests/translator_checkpoints.rs index 669756a0..fd937b10 100644 --- a/bt-daemon/tests/translator_checkpoints.rs +++ b/bt-daemon/tests/translator_checkpoints.rs @@ -3,29 +3,11 @@ clippy::expect_used, reason = "Checkpoint regression fixtures fail immediately." )] -#[path = "support/distributed.rs"] -#[allow(dead_code)] -mod distributed; +#[path = "support/translation_streams.rs"] +mod streams; use bt_daemon::wire::Envelope; use bt_daemon::{AgentTranslator, Registry, SessionCtx, SpanOp}; -use distributed::{AgentKind, DistributedFixtures, ProcessTree}; -use serde_json::{json, Value}; -fn event(source: &str, name: &str, ts: i64, payload: Value) -> Envelope { - Envelope { - source: source.into(), - source_version: None, - plugin_version: None, - session_id: "checkpoint".into(), - event: name.into(), - ts_ms: ts, - managed_run_id: None, - capture: None, - payload, - route: None, - config: None, - } -} fn collect(translator: &mut dyn AgentTranslator, env: &Envelope, ctx: &SessionCtx) -> Vec { let mut ops = translator.handle(env, ctx).unwrap(); while let Some(batch) = translator.drain_pending(ctx).unwrap() { @@ -89,114 +71,41 @@ fn assert_stream(source: &str, events: Vec) { } #[test] fn checkpoints_preserve_native_turns_tools_transcripts_and_later_turn_ids() { - for kind in AgentKind::ALL { - let mut fixtures = DistributedFixtures::new(); - let tree = ProcessTree::root(1234); - let mut events = fixtures.start_turn(kind, "checkpoint", &tree, "first", 1_700_000_000_000); - events.push(fixtures.open_tool( - kind, - "checkpoint", - &tree, - "call", - "work", - 1_700_000_000_010, - )); - events.push(fixtures.close_tool( - kind, - "checkpoint", - &tree, - "call", - "work", - "result", - 1_700_000_000_020, - )); - events.push(fixtures.close_session( - kind, - "checkpoint", - &tree, - "complete", - 1_700_000_000_030, - )); - events.extend(fixtures.start_turn(kind, "checkpoint", &tree, "second", 1_700_000_000_040)); - let source = events[0].source.clone(); - assert_stream(&source, events); + for (source, events, _fixtures) in streams::native_streams() { + if ["claude-code", "codex", "pi", "opencode"].contains(&source.as_str()) { + assert_stream(&source, events); + } } } #[test] fn muse_checkpoints_preserve_open_operations_and_cumulative_usage() { - let events = include_str!("fixtures/muse-hooks-1.1.1.jsonl") - .lines() - .enumerate() - .map(|(i, line)| { - let payload: Value = serde_json::from_str(line).unwrap(); - let name = payload["hook_event_name"].as_str().unwrap().to_owned(); - event("muse", &name, i as i64 + 1, payload) - }) - .collect(); - assert_stream("muse", events); + for (source, events, _fixtures) in streams::native_streams() { + if source == "muse" { + assert_stream(&source, events); + } + } } #[test] fn cursor_checkpoints_preserve_open_tools_and_internal_span_fields() { - let events = [ - ("sessionStart", json!({})), - ("beforeSubmitPrompt", json!({"prompt":"work"})), - ( - "preToolUse", - json!({"tool_name":"Shell","tool_use_id":"call","tool_input":{"command":"echo hello"}}), - ), - ( - "postToolUse", - json!({"tool_name":"Shell","tool_use_id":"call","tool_output":"hello","tool_input":{"command":"echo hello"},"duration":1}), - ), - ("afterAgentResponse", json!({"text":"done"})), - ("stop", json!({"status":"completed"})), - ("beforeSubmitPrompt", json!({"prompt":"next"})), - ] - .into_iter() - .enumerate() - .map(|(i, (name, mut payload))| { - payload["hook_event_name"] = name.into(); - payload["conversation_id"] = "checkpoint".into(); - payload["generation_id"] = format!("generation-{i}").into(); - payload["model"] = "test".into(); - payload["workspace_roots"] = json!(["/workspace"]); - event("cursor", name, i as i64 + 1, payload) - }) - .collect(); - assert_stream("cursor", events); + for (source, events, _fixtures) in streams::native_streams() { + if source == "cursor" { + assert_stream(&source, events); + } + } } #[test] fn antigravity_checkpoints_preserve_invocations_across_native_boundaries() { - let records = [ - json!({"step_index":0,"source":"USER_EXPLICIT","type":"USER_INPUT","status":"COMPLETED","content":"List files"}), - json!({"step_index":1,"source":"MODEL","type":"PLANNER_RESPONSE","status":"COMPLETED","content":"Inspecting","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"}}],"usage":{"inputTokens":12,"outputTokens":7,"totalTokens":19}}), - json!({"step_index":2,"source":"MODEL","type":"CORTEX_STEP_TYPE_RUN_COMMAND","status":"COMPLETED","content":"README","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"},"result":"README"}]}), - ]; - let mut contents = String::new(); - let mut boundaries = Vec::new(); - for record in records { - contents.push_str(&serde_json::to_string(&record).unwrap()); - contents.push('\n'); - boundaries.push(contents.len()); + for (source, events, _fixtures) in streams::native_streams() { + if source == "antigravity" { + assert_stream(&source, events); + } } - let events = [("PreInvocation", boundaries[0]), ("PostInvocation", boundaries[1]), ("PostToolUse", boundaries[2]), ("Stop", boundaries[2])].into_iter().enumerate().map(|(i, (name, through))| { - event("antigravity", name, i as i64 + 1, json!({"conversationId":"checkpoint","invocationNum":0,"initialNumSteps":1,"stepIdx":2,"fullyIdle":true, - "transcriptPath":"/tmp/checkpoint/transcript.jsonl", - "_bt_transcript_observation":{"path":"/tmp/checkpoint/transcript.jsonl","observed_bytes":through,"full_path":"/tmp/checkpoint/transcript_full.jsonl","full_observed_bytes":through}, - "_bt_transcript_snapshot":{"path":"/tmp/checkpoint/transcript_full.jsonl","contents":contents}})) - }).collect(); - assert_stream("antigravity", events); } #[test] fn grok_checkpoints_preserve_transcript_cursors_and_pending_model_work() { - let root = - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/grok/transcript"); - let updates = root.join("updates.jsonl"); - let events = root.join("events.jsonl"); - let mut offset = 0; - let stream = std::fs::read(&updates).unwrap().split_inclusive(|byte| *byte == b'\n').map(|line| { - offset += line.len() as u64; - event("grok", "Wake", 1600, json!({"cwd":"/repo","_bt_grok_transcript_mirrors":{"updates":{"mirror":updates,"through":offset},"events":{"mirror":events,"through":std::fs::metadata(&events).unwrap().len()}}})) - }).collect(); - assert_stream("grok", stream); + for (source, events, _fixtures) in streams::native_streams() { + if source == "grok" { + assert_stream(&source, events); + } + } } From 1c1a60e1f26d508712b5e0ac61b4a286859a1cde Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 3 Nov 2026 01:12:10 +0800 Subject: [PATCH 13/19] Use one bounded capture queue during startup and normal operation --- bt-daemon/src/journal.rs | 180 ++-------------- bt-daemon/src/journal/codec.rs | 34 +-- bt-daemon/src/pipeline/input.rs | 36 +--- bt-daemon/src/pipeline/translation.rs | 149 ++++--------- bt-daemon/src/server/ingress.rs | 299 ++++---------------------- bt-daemon/src/server/maintenance.rs | 122 ++--------- bt-daemon/src/server/mod.rs | 19 +- bt-daemon/src/server/recovery.rs | 9 +- bt-daemon/src/server/sessions.rs | 10 - 9 files changed, 149 insertions(+), 709 deletions(-) diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index acad9326..e2d3d475 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -820,14 +820,6 @@ pub fn envelope_from_redacted(r: RedactedEnvelope) -> Envelope { #[cfg(test)] mod tests { use super::*; - mod streams { - use crate as bt_daemon; - include!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/support/translation_streams.rs" - )); - } - fn pi_context(messages: Vec, ts_ms: i64) -> Envelope { Envelope { source: "pi".into(), @@ -844,104 +836,6 @@ mod tests { } } - #[tokio::test] - async fn compressed_recovery_matches_live_native_translation_for_every_source() { - use crate::translate::{AgentTranslator, Registry, SessionCtx, SpanOp}; - fn collect( - translator: &mut dyn AgentTranslator, - env: &Envelope, - ctx: &SessionCtx, - ) -> Vec { - let mut ops = translator.handle(env, ctx).unwrap(); - while let Some(batch) = translator.drain_pending(ctx).unwrap() { - ops.extend(batch); - } - comparable(ops) - } - fn comparable(ops: Vec) -> Vec { - let mut values = ops - .into_iter() - .map(|op| { - let (SpanOp::Insert(row) | SpanOp::Merge(row)) = &op; - serde_json::to_string(&( - &op, - &row.turn_span_id, - row.is_turn, - &row.late_merge_key, - &row.late_merge_version, - )) - .unwrap() - }) - .collect::>(); - values.sort(); - values - } - let registry = Registry::default_agents(); - let mut covered = Vec::new(); - for (source, mut events, _fixtures) in streams::native_streams() { - covered.push(source.clone()); - // A retried first hook is valid native input and guarantees an - // actual delta for every agent, even a short unique-kind fixture. - events.insert(0, events[0].clone()); - for event in &mut events { - event.payload["journal_codec_padding"] = - serde_json::json!("é📚shared history".repeat(1024)); - } - let tmp = tempfile::tempdir().unwrap(); - let path = tmp.path().join("events.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let mut tail = 0; - for event in &events { - tail = writer.append(event).await.unwrap(); - } - assert!(tokio::fs::read_to_string(&path) - .await - .unwrap() - .contains(codec::DELTA_RECORD)); - let session = events[0].session_id.clone(); - let ctx = SessionCtx { - session_id: session.clone(), - config: None, - }; - let mut live = registry.create(&source, &session); - let mut recovered = registry.create(&source, &session); - let mut reader = JournalReader::open(&path, tail).await.unwrap().unwrap(); - for (index, original) in events.iter().enumerate() { - let replayed = reader.next_entry().await.unwrap().unwrap(); - assert_eq!( - serde_json::to_value(&replayed).unwrap(), - serde_json::to_value(original.redacted()).unwrap(), - "{source} payload {index}" - ); - let replayed = envelope_from_redacted(replayed); - assert_eq!( - collect(live.as_mut(), original, &ctx), - collect(recovered.as_mut(), &replayed, &ctx), - "{source} event {index}" - ); - // Restart the decoder at every boundary, using only its saved - // cache. No previous record may be needed to expand the next. - let saved = serde_json::to_vec(reader.decoder_context()).unwrap(); - reader = JournalReader::resume( - &path, - reader.position, - tail, - serde_json::from_slice(&saved).unwrap(), - ) - .await - .unwrap(); - } - assert!(reader.next_entry().await.unwrap().is_none()); - assert_eq!( - comparable(live.finalize(&ctx).unwrap()), - comparable(recovered.finalize(&ctx).unwrap()), - "{source} finalization" - ); - } - covered.sort(); - assert_eq!(covered, registry.sources()); - } - #[tokio::test] async fn delta_recovery_uses_original_json_bytes_not_reader_serialization() { let tmp = tempfile::tempdir().unwrap(); @@ -1037,61 +931,25 @@ mod tests { } #[tokio::test] - async fn every_source_journals_complete_payloads_in_event_order() { - for source in crate::translate::Registry::default_agents().sources() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("events.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let mut expected = Vec::new(); - for (index, name) in [ - "context", - "before_provider_request", - "message_update", - "agent_end", - ] - .into_iter() - .enumerate() - { - let mut event = pi_context( - vec![serde_json::json!({"role":"user","content":"entire history"})], - index as i64, - ); - event.source = source.clone(); - event.event = name.into(); - event.payload["event"]["type"] = name.into(); - event.payload["event"]["message"] = - serde_json::json!({"role":"assistant","content":"growing message"}); - event.payload["event"]["assistantMessageEvent"] = serde_json::json!({"type":"text_delta","delta":"token","partial":"partial message"}); - event.payload["event"]["payload"] = serde_json::json!({"system":"system prompt","messages":[{"role":"user","content":"provider messages"}],"unknown_provider_field":{"preserve":true}}); - writer.append(&event).await.unwrap(); - expected.push(event); - } - drop(writer); - let stored = tokio::fs::read_to_string(&path).await.unwrap(); - let records = stored - .lines() - .map(|line| serde_json::from_str::(line).unwrap()) - .collect::>(); - let mut reader = JournalReader::open(&path, stored.len() as u64) - .await - .unwrap() - .unwrap(); - for (index, event) in expected.iter().enumerate() { - assert_eq!( - records[index]["payload"], event.payload, - "{source}: {} was changed before persistence", - event.event - ); - let replayed = reader.next_entry().await.unwrap().unwrap(); - assert_eq!(replayed.event, event.event); - assert_eq!(replayed.payload, event.payload); - assert!( - reader.decoder_context().legacy_pi.is_empty(), - "new {source} records must not require the legacy Pi message decoder" - ); - } - assert!(reader.next_entry().await.unwrap().is_none()); - } + async fn small_payloads_preserve_unknown_native_fields_without_deltas() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let mut event = pi_context( + vec![serde_json::json!({"role":"user","content":"history"})], + 1, + ); + event.payload["unknown_provider_field"] = serde_json::json!({"preserve":true}); + let tail = writer.append(&event).await.unwrap(); + let stored: serde_json::Value = + serde_json::from_str(&tokio::fs::read_to_string(&path).await.unwrap()).unwrap(); + assert_eq!(stored["payload"], event.payload); + let mut reader = JournalReader::open(&path, tail).await.unwrap().unwrap(); + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + event.payload + ); + assert!(reader.decoder_context().legacy_pi.is_empty()); } #[tokio::test] diff --git a/bt-daemon/src/journal/codec.rs b/bt-daemon/src/journal/codec.rs index fa039695..9892db6e 100644 --- a/bt-daemon/src/journal/codec.rs +++ b/bt-daemon/src/journal/codec.rs @@ -217,26 +217,17 @@ impl DecoderContext { impl<'de> Deserialize<'de> for DecoderContext { fn deserialize>(deserializer: D) -> Result { #[derive(Deserialize)] - #[serde(untagged)] - enum Stored { - Legacy(Vec), - Current { - legacy_pi: Vec, - payloads: PayloadCache, - }, + struct Stored { + legacy_pi: Vec, + payloads: PayloadCache, } - let context = match Stored::deserialize(deserializer)? { - Stored::Legacy(legacy_pi) => Self { - legacy_pi, - payloads: PayloadCache::default(), - }, - Stored::Current { - legacy_pi, - payloads, - } => Self { - legacy_pi, - payloads, - }, + let Stored { + legacy_pi, + payloads, + } = Stored::deserialize(deserializer)?; + let context = Self { + legacy_pi, + payloads, }; context .payloads @@ -300,10 +291,7 @@ mod tests { } #[test] - fn checkpoints_read_legacy_decoder_state_and_preserve_shared_bases() { - let legacy = vec![json!({"role":"user","content":"legacy"})]; - let context: DecoderContext = serde_json::from_value(json!(legacy)).unwrap(); - assert_eq!(context.legacy_pi, legacy); + fn checkpoints_preserve_shared_bases() { let mut context = DecoderContext::default(); let payload = json!({"messages":["content".repeat(1000)]}); let (json, _) = context.payloads.encode("message", &payload).unwrap(); diff --git a/bt-daemon/src/pipeline/input.rs b/bt-daemon/src/pipeline/input.rs index 61b94a97..f7548d1e 100644 --- a/bt-daemon/src/pipeline/input.rs +++ b/bt-daemon/src/pipeline/input.rs @@ -135,36 +135,6 @@ mod tests { } } - #[test] - fn every_agent_can_supply_original_events_without_any_journal_adapter() { - for source in crate::translate::Registry::default_agents().sources() { - let input = SourceInput::default(); - let first = event(&source, 0, 16 * 1024); - let first_position = position(0, &first); - let boundary = first_position.through; - input.push_event(first.clone(), first_position); - let second = event(&source, 1, 16 * 1024); - let second_position = position(boundary, &second); - let through = second_position.through; - input.push_event(second.clone(), second_position); - let record = input.take(0, boundary).unwrap(); - let SourceEvent::Event(actual) = record.event else { - panic!("expected event") - }; - assert_eq!(actual.payload, first.payload); - assert!( - input.take(boundary, boundary).is_none(), - "must not outrun the requested boundary" - ); - let record = input.take(boundary, through).unwrap(); - let SourceEvent::Event(actual) = record.event else { - panic!("expected event") - }; - assert_eq!(actual.payload, second.payload); - assert!(input.is_empty()); - } - } - #[test] fn oversized_events_flow_directly_but_eviction_keeps_the_input_bounded() { let input = SourceInput::default(); @@ -172,7 +142,11 @@ mod tests { let first = position(0, &oversized); let boundary = first.through; input.push_event(oversized.clone(), first); - assert!(input.take(0, boundary).is_some()); + assert!(input.take(0, boundary - 1).is_none()); + let SourceEvent::Event(actual) = input.take(0, boundary).unwrap().event else { + panic!("expected original event"); + }; + assert_eq!(actual.payload, oversized.payload); input.push_event(oversized.clone(), position(boundary, &oversized)); let gap_end = position(boundary, &oversized).through; let next = event("debug", 1, 3 * 1024 * 1024); diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 7f8fe04f..503a4fb2 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -1107,18 +1107,16 @@ mod tests { rows.sort(); rows } + let mut covered = Vec::new(); for (source, mut events, _fixtures) in streams::native_streams() { + covered.push(source.clone()); events.insert(0, events[0].clone()); for event in &mut events { event.payload["codec_padding"] = serde_json::json!("é📚history".repeat(1024)); } let mut expected = None; for mode in ["live", "overflow", "cold"] { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), &source, "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, path, writer) = source_journal(&source).await; let options = resume_options(tmp.path(), &source, writer.clone()); let input = options.input.clone(); let mut translation = Translation::spawn(options).await.unwrap(); @@ -1141,6 +1139,14 @@ mod tests { translation.advance(positions[0]).await.unwrap(); translation.barrier().await.unwrap(); translation.shutdown().await; + // A restart must seek over this prefix, not replay it. + let mut bytes = tokio::fs::read(&path).await.unwrap(); + for byte in &mut bytes[..positions[0] as usize] { + if *byte != b'\n' { + *byte = b'x'; + } + } + tokio::fs::write(&path, bytes).await.unwrap(); // New writer and input: the following stored delta must // rely on the SQLite decoder/reducer checkpoint alone. *writer.lock().await = JournalWriter::open_path(&path).await.unwrap(); @@ -1168,6 +1174,23 @@ mod tests { translation.shutdown().await; } } + covered.sort(); + assert_eq!(covered, Registry::default_agents().sources()); + } + + async fn source_journal( + source: &str, + ) -> ( + tempfile::TempDir, + PathBuf, + Arc>, + ) { + let dir = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(dir.path(), source, "checkpoint"); + let writer = Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )); + (dir, path, writer) } fn resume_options( @@ -1192,11 +1215,7 @@ mod tests { #[tokio::test] async fn checkpoints_cannot_absorb_captured_events_before_dispatch() { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, _, writer) = source_journal("debug").await; let options = resume_options(tmp.path(), "debug", writer.clone()); let input = options.input.clone(); let source = Translation::spawn(options).await.unwrap(); @@ -1224,11 +1243,7 @@ mod tests { #[tokio::test] async fn live_translation_and_controls_do_not_read_events_back_from_disk() { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, path, writer) = source_journal("debug").await; let options = resume_options(tmp.path(), "debug", writer.clone()); let input = options.input.clone(); let translation = Translation::spawn(options).await.unwrap(); @@ -1299,11 +1314,7 @@ mod tests { #[tokio::test] async fn evicted_input_recovers_only_the_gap_then_rejoins_direct_capture() { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, path, writer) = source_journal("debug").await; let options = resume_options(tmp.path(), "debug", writer.clone()); let input = options.input.clone(); let translation = Translation::spawn(options).await.unwrap(); @@ -1376,11 +1387,7 @@ mod tests { #[tokio::test] async fn synchronous_replay_work_cannot_block_the_hook_runtime() { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, _, writer) = source_journal("debug").await; let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{}})).unwrap(); let through = writer.lock().await.append(&event).await.unwrap(); let options = resume_options(tmp.path(), "debug", writer.clone()); @@ -1470,11 +1477,7 @@ mod tests { #[tokio::test] async fn slow_checkpoint_encoding_does_not_block_journal_capture_on_the_runtime() { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, path, writer) = source_journal("debug").await; let entered = Arc::new(Notify::new()); let (release, resume) = std::sync::mpsc::channel(); let mut producer = Producer { @@ -1524,11 +1527,7 @@ mod tests { #[tokio::test] async fn checkpoint_budget_counts_decoded_payloads_after_compression() { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, _, writer) = source_journal("debug").await; let source = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) .await .unwrap(); @@ -1559,11 +1558,7 @@ mod tests { #[tokio::test] async fn checkpoint_window_accounts_for_committed_suffix_on_restart() { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, _, writer) = source_journal("debug").await; let source = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) .await .unwrap(); @@ -1605,77 +1600,11 @@ mod tests { resumed.shutdown().await; } - #[tokio::test] - async fn every_agent_journal_resumes_at_its_persisted_byte_offset() { - for source in Registry::default_agents().sources() { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), &source, "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); - let translation = - Translation::spawn(resume_options(tmp.path(), &source, writer.clone())) - .await - .unwrap(); - let control = PipelineControl { - kind: ControlKind::Checkpoint, - bounds: HashMap::new(), - }; - let mut tail = 0; - // Ordered controls exercise the real producer and each concrete - // translator without depending on a particular native hook schema. - // Native open-span/state equivalence is covered separately. - for _ in 0..CHECKPOINT_EVENTS + 3 { - tail = writer.lock().await.append_control(&control).await.unwrap(); - } - translation.advance(tail).await.unwrap(); - translation.barrier().await.unwrap(); - let checkpoint = translation - .journal - .translation_checkpoint() - .await - .unwrap() - .unwrap(); - assert!( - checkpoint.input > 0 && checkpoint.input < tail, - "{source}: periodic checkpoint must precede the uncheckpointed suffix" - ); - let task = translation.task.lock().await.take().unwrap(); - task.abort(); - let _ = task.await; - let mut contents = tokio::fs::read(&path).await.unwrap(); - for byte in &mut contents[..checkpoint.input as usize] { - if *byte != b'\n' { - *byte = b'x'; - } - } - tokio::fs::write(&path, contents).await.unwrap(); - tail = writer.lock().await.append_control(&control).await.unwrap(); - let resumed = Translation::spawn(resume_options(tmp.path(), &source, writer.clone())) - .await - .unwrap(); - resumed.advance(tail).await.unwrap(); - resumed.barrier().await.unwrap_or_else(|error| { - panic!("{source} could not resume past its saved byte offset: {error}") - }); - assert_eq!( - resumed.journal.translated_through().await.unwrap(), - tail, - "{source}" - ); - resumed.shutdown().await; - } - } - #[tokio::test] async fn checkpoint_seeks_past_history_for_native_and_legacy_journals() { for format in ["debug", "pi", "legacy-pi"] { let source = if format == "legacy-pi" { "pi" } else { format }; - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), source, "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, path, writer) = source_journal(source).await; let translation = Translation::spawn(resume_options(tmp.path(), source, writer.clone())) .await @@ -1775,11 +1704,7 @@ mod tests { "UPDATE translation_checkpoint SET state=x'00'", "UPDATE translation_checkpoint SET version='old'", ] { - let tmp = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); + let (tmp, _, writer) = source_journal("debug").await; let env: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ "source":"debug", "session_id":"checkpoint", "event":"event", "ts_ms":1,"payload":{}, })).unwrap(); diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 77f0f8d2..8088b694 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -4,9 +4,9 @@ use super::*; use std::collections::VecDeque; -/// Live startup captures remain ordered and bounded. Evicted entries are +/// Captures remain ordered and bounded. Evicted entries are /// already durable; their source journal's high-water mark is replayed first. -pub(super) struct StartupBuffer { +pub(super) struct CaptureBuffer { pub(super) events: VecDeque, pub(super) evicted: HashMap, } @@ -17,7 +17,7 @@ pub(super) struct EvictedRange { pub(super) through: u64, } -impl StartupBuffer { +impl CaptureBuffer { pub(super) fn new() -> Self { Self { events: VecDeque::new(), @@ -46,38 +46,32 @@ impl StartupBuffer { } } -pub(super) enum StartupWork { +pub(super) enum CaptureWork { Evicted(HashMap), Event(Box), Finished, } impl Daemon { - pub(super) fn next_startup_work(&self) -> StartupWork { - let mut state = self.startup_buffer.lock(); - let Some(buffer) = state.as_mut() else { - return StartupWork::Finished; - }; + pub(super) fn next_capture_work(&self) -> CaptureWork { + let mut buffer = self.capture_buffer.lock(); if !buffer.evicted.is_empty() { - return StartupWork::Evicted(std::mem::take(&mut buffer.evicted)); + return CaptureWork::Evicted(std::mem::take(&mut buffer.evicted)); } if let Some(event) = buffer.events.pop_front() { - return StartupWork::Event(Box::new(event)); + return CaptureWork::Event(Box::new(event)); } - *state = None; - StartupWork::Finished + CaptureWork::Finished } } -pub(super) async fn drain_startup_buffer(daemon: &Arc) -> anyhow::Result<()> { +pub(super) async fn drain_capture_buffer(daemon: &Arc) -> anyhow::Result<()> { let mut cursors = HashMap::new(); loop { - match daemon.next_startup_work() { - StartupWork::Evicted(bounds) => { - replay_evicted_startup(daemon, &mut cursors, &bounds).await? - } - StartupWork::Event(event) => dispatch_ingress_event(daemon, *event).await, - StartupWork::Finished => return Ok(()), + match daemon.next_capture_work() { + CaptureWork::Evicted(bounds) => replay_evicted(daemon, &mut cursors, &bounds).await?, + CaptureWork::Event(event) => dispatch_ingress_event(daemon, *event).await, + CaptureWork::Finished => return Ok(()), } } } @@ -90,7 +84,7 @@ pub(super) struct ReplayCursor { /// Advance each evicted journal range from its previous position, retaining /// the reader's payload decoder context between advances. Older records are decoded /// only to initialize that context; they are not dispatched again. -pub(super) async fn replay_evicted_startup( +pub(super) async fn replay_evicted( daemon: &Arc, cursors: &mut HashMap, bounds: &HashMap, @@ -98,7 +92,7 @@ pub(super) async fn replay_evicted_startup( for (path, range) in bounds { anyhow::ensure!( journal::JournalReader::recorded_len(path).await >= range.through, - "evicted startup journal is truncated" + "evicted capture journal is truncated" ); if !cursors.contains_key(path) { let (reader, position) = match resume_discovery(daemon, path, range.from).await { @@ -106,7 +100,7 @@ pub(super) async fn replay_evicted_startup( _ => ( journal::JournalReader::open(path, range.from) .await? - .ok_or_else(|| anyhow::anyhow!("evicted startup journal disappeared"))?, + .ok_or_else(|| anyhow::anyhow!("evicted capture journal disappeared"))?, 0, ), }; @@ -114,22 +108,22 @@ pub(super) async fn replay_evicted_startup( } let cursor = cursors .get_mut(path) - .ok_or_else(|| anyhow::anyhow!("startup cursor missing"))?; + .ok_or_else(|| anyhow::anyhow!("recovery cursor missing"))?; anyhow::ensure!( cursor.position <= range.from, - "evicted startup ranges overlap" + "evicted capture ranges overlap" ); cursor.reader.extend(range.from); while cursor.position < range.from { let entry = cursor.reader.next_record().await?.ok_or_else(|| { - anyhow::anyhow!("evicted startup range begins inside a record") + anyhow::anyhow!("evicted capture range begins inside a record") })?; cursor.position = entry.through; } anyhow::ensure!( cursor.position == range.from, - "invalid evicted startup boundary" + "invalid evicted capture boundary" ); cursor.reader.extend(range.through); } @@ -139,7 +133,7 @@ pub(super) async fn replay_evicted_startup( for (path, range) in bounds { let cursor = cursors .get_mut(path) - .ok_or_else(|| anyhow::anyhow!("startup cursor missing"))?; + .ok_or_else(|| anyhow::anyhow!("recovery cursor missing"))?; if let Some((sequence, event)) = next_evicted_event(daemon, cursor, range.through).await? { ready.push((sequence, path.clone(), event)); } @@ -150,12 +144,12 @@ pub(super) async fn replay_evicted_startup( .enumerate() .min_by_key(|(_, (sequence, _, _))| *sequence) .map(|(index, _)| index) - .ok_or_else(|| anyhow::anyhow!("startup lookahead missing"))?; + .ok_or_else(|| anyhow::anyhow!("recovery lookahead missing"))?; let (_, path, event) = ready.swap_remove(index); dispatch_ingress_event(daemon, event).await; let cursor = cursors .get_mut(&path) - .ok_or_else(|| anyhow::anyhow!("startup cursor missing"))?; + .ok_or_else(|| anyhow::anyhow!("recovery cursor missing"))?; if let Some((sequence, event)) = next_evicted_event(daemon, cursor, bounds[&path].through).await? { @@ -170,19 +164,6 @@ async fn next_evicted_event( cursor: &mut ReplayCursor, through: u64, ) -> anyhow::Result> { - let Some((sequence, event)) = next_replayed_event(daemon, cursor, through).await? else { - return Ok(None); - }; - let sequence = - sequence.ok_or_else(|| anyhow::anyhow!("live startup event has no capture sequence"))?; - Ok(Some((sequence, event))) -} - -async fn next_replayed_event( - daemon: &Daemon, - cursor: &mut ReplayCursor, - through: u64, -) -> anyhow::Result, PendingEvent)>> { while cursor.position < through { let before = cursor.position; let entry = cursor @@ -199,7 +180,9 @@ async fn next_replayed_event( }; env.source = source.to_owned(); return Ok(Some(( - entry.capture_sequence, + entry + .capture_sequence + .ok_or_else(|| anyhow::anyhow!("captured event has no sequence"))?, PendingEvent { env, replay_through: before, @@ -214,7 +197,7 @@ async fn next_replayed_event( } pub(super) enum IngressMsg { - Event(Box), + Wake, Barrier(oneshot::Sender<()>), Stop(oneshot::Sender<()>), } @@ -236,32 +219,24 @@ pub(super) fn spawn_ingress_worker( return; } } - loop { - if daemon.ingress_overflow.swap(false, Ordering::AcqRel) { - recover_ingress_overflow(&daemon).await; - continue; + while let Some(msg) = rx.recv().await { + // The same queue drains before barriers and after ordinary wakes. + // Capture never waits for its reader, actors, or destinations. + if let Err(error) = drain_capture_buffer(&daemon).await { + tracing::error!(%error, "capture recovery failed; stopping ingress"); + daemon.trigger_shutdown(); + return; } - let Some(msg) = rx.recv().await else { - break; - }; match msg { - IngressMsg::Stop(reply) => { + IngressMsg::Wake => {} + IngressMsg::Barrier(reply) => { + daemon.settle_session_actors().await; + let _ = retry_pending_sessions(&daemon).await; let _ = reply.send(()); - break; - } - IngressMsg::Event(event) => { - dispatch_ingress_event(&daemon, *event).await; } - IngressMsg::Barrier(reply) => { - loop { - daemon.settle_session_actors().await; - let _ = retry_pending_sessions(&daemon).await; - if !daemon.ingress_overflow.swap(false, Ordering::AcqRel) { - break; - } - recover_ingress_overflow(&daemon).await; - } + IngressMsg::Stop(reply) => { let _ = reply.send(()); + break; } } } @@ -269,20 +244,6 @@ pub(super) fn spawn_ingress_worker( } pub(super) async fn dispatch_ingress_event(daemon: &Arc, event: PendingEvent) { - let ingress_key = - event.env.route.as_ref().and_then(|route| { - DeliveryKey::new(&event.env.source, &event.env.session_id, route).ok() - }); - if ingress_key.as_ref().is_some_and(|key| { - daemon - .ingress_dispatched - .lock() - .get(key) - .is_some_and(|through| *through >= event.journal_through) - }) { - return; - } - // Startup discovery rebuilds these observations before this worker is // released, so a historical event cannot overwrite a newer capture. daemon.remember_claude_transcripts(&event.env); @@ -298,17 +259,8 @@ pub(super) async fn dispatch_ingress_event(daemon: &Arc, event: PendingE daemon.settle_session_actors().await; } let source_env = event.env.clone(); - match accept_event(daemon, event).await { - Ok(()) => { - if let Some(key) = ingress_key { - let mut dispatched = daemon.ingress_dispatched.lock(); - let through = dispatched.entry(key).or_default(); - *through = (*through).max(journal_through); - } - } - Err(error) => { - tracing::warn!(%error, "journaled ingress event could not be dispatched"); - } + if let Err(error) = accept_event(daemon, event).await { + tracing::warn!(%error, "journaled ingress event could not be dispatched"); } // Authentication and route discovery do not prevent canonical persistence. match daemon.translation_for(&source_env).await { @@ -323,154 +275,6 @@ pub(super) async fn dispatch_ingress_event(daemon: &Arc, event: PendingE } } -struct OverflowCursor { - path: PathBuf, - replay: ReplayCursor, - through: u64, - acknowledged: HashMap, -} - -type OverflowEvent = ((bool, u64), PendingEvent); - -async fn next_overflow_event( - daemon: &Arc, - cursor: &mut OverflowCursor, -) -> Option { - loop { - let (sequence, event) = match next_replayed_event( - daemon, - &mut cursor.replay, - cursor.through, - ) - .await - { - Ok(Some(event)) => event, - Ok(None) => return None, - Err(error) => { - tracing::warn!(%error, path=%cursor.path.display(), "ingress overflow replay failed"); - return None; - } - }; - if event.journal_through - <= daemon - .ingress_history - .lock() - .get(&cursor.path) - .copied() - .unwrap_or(0) - { - continue; - } - let Some(route) = event.env.route.as_ref() else { - continue; - }; - let Ok(key) = DeliveryKey::new(&event.env.source, &event.env.session_id, route) else { - continue; - }; - let dispatched = daemon - .ingress_dispatched - .lock() - .get(&key) - .copied() - .unwrap_or(0); - if event.journal_through <= dispatched { - continue; - } - let route_json = recovered_delivery_route(daemon, &event.env) - .await - .as_ref() - .and_then(|route| serde_json::to_string(route).ok()) - .unwrap_or_else(|| serde_json::to_string(route).unwrap_or_default()); - if event.journal_through <= cursor.acknowledged.get(&route_json).copied().unwrap_or(0) { - continue; - } - return Some(((sequence.is_some(), sequence.unwrap_or(0)), event)); - } -} - -/// Recover a frozen durable range, merging one lookahead per source by capture -/// ordinal. Per-file replay would close a parent's tool before dispatching a -/// child captured between that tool's start and completion. -pub(super) async fn recover_ingress_overflow(daemon: &Arc) { - let Ok(mut entries) = tokio::fs::read_dir(journal::journal_dir(&daemon.data_dir)).await else { - return; - }; - let mut ready = Vec::new(); - while let Ok(Some(entry)) = entries.next_entry().await { - let path = entry.path(); - if path.extension().and_then(|ext| ext.to_str()) != Some("ndjson") { - continue; - } - let through = daemon.journal_snapshot(&path).await; - if through - <= daemon - .ingress_history - .lock() - .get(&path) - .copied() - .unwrap_or(0) - { - continue; - } - let mut acknowledged = HashMap::::new(); - let (reader, start) = if let Ok(Some((reader, position, _))) = - resume_discovery(daemon, &path, through).await - { - // A compatible checkpoint already uses SQLite delivery heads. - (reader, position) - } else { - let Ok(Some(mut legacy)) = journal::JournalReader::open(&path, through).await else { - continue; - }; - while let Ok(Some(entry)) = legacy.next_record().await { - if let journal::JournalRecord::DeliveryCheckpoint { route, through } = entry.record - { - let key = serde_json::to_string(&route).unwrap_or_default(); - let head = acknowledged.entry(key).or_default(); - *head = (*head).max(through); - } - } - let Ok(Some(reader)) = journal::JournalReader::open(&path, through).await else { - continue; - }; - (reader, 0) - }; - let mut cursor = OverflowCursor { - path, - replay: ReplayCursor { - reader, - position: start, - }, - through, - acknowledged, - }; - if let Some((order, event)) = next_overflow_event(daemon, &mut cursor).await { - ready.push((order, event, cursor)); - } - } - let mut recovered = 0; - while !ready.is_empty() { - let Some(index) = ready - .iter() - .enumerate() - .min_by_key(|(_, (order, _, _))| *order) - .map(|(index, _)| index) - else { - break; - }; - let (_, event, mut cursor) = ready.swap_remove(index); - debug_assert!(event.journal_through <= cursor.through); - dispatch_ingress_event(daemon, event).await; - recovered += 1; - if let Some((order, event)) = next_overflow_event(daemon, &mut cursor).await { - ready.push((order, event, cursor)); - } - } - if recovered > 0 { - tracing::info!(recovered, "drained journal-backed ingress overflow"); - } -} - pub(super) fn spawn_pending_reconciler(daemon: Arc) { tokio::spawn(async move { let mut tick = tokio::time::interval(Duration::from_millis(100)); @@ -629,20 +433,9 @@ impl Daemon { journal_through, settings_through: None, }; - let event = { - let mut state = self.startup_buffer.lock(); - if let Some(buffer) = state.as_mut() { - buffer.push(&self.data_dir, event); - return Ok(true); - } - event - }; - match self.ingress_tx.try_send(IngressMsg::Event(Box::new(event))) { - Ok(()) => {} - Err(mpsc::error::TrySendError::Full(_)) => { - self.ingress_overflow.store(true, Ordering::Release); - tracing::debug!("ingress queue full; journal will be drained by daemon worker"); - } + self.capture_buffer.lock().push(&self.data_dir, event); + match self.ingress_tx.try_send(IngressMsg::Wake) { + Ok(()) | Err(mpsc::error::TrySendError::Full(_)) => {} Err(mpsc::error::TrySendError::Closed(_)) => { tracing::warn!("journaled event will be recovered after daemon restart"); } diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 55f92f75..6c7d5dba 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -131,11 +131,6 @@ async fn collect_partitions(daemon: &Daemon, age: Duration) { if tokio::fs::remove_file(&storage).await.is_err() { continue; } - daemon.ingress_history.lock().remove(&path); - daemon - .ingress_dispatched - .lock() - .retain(|key, _| key.source != first.source || key.session_id != first.session_id); for sidecar in [wal, storage.with_extension("sqlite-shm"), path] { if let Err(error) = tokio::fs::remove_file(&sidecar).await { if error.kind() != std::io::ErrorKind::NotFound { @@ -504,7 +499,7 @@ mod tests { .unwrap() .contains("during-startup")); assert!(daemon.translations.lock().is_empty()); - drain_startup_buffer(&daemon).await.unwrap(); + drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); tokio::time::timeout(Duration::from_secs(3), daemon.ingress_barrier()) .await @@ -537,7 +532,7 @@ mod tests { tokio::task::yield_now().await; } tokio::task::yield_now().await; - drain_startup_buffer(&daemon).await.unwrap(); + drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); tokio::time::timeout(Duration::from_secs(10), shutdown) .await @@ -575,7 +570,7 @@ mod tests { for event in recovered { dispatch_ingress_event(&daemon, event).await; } - drain_startup_buffer(&daemon).await.unwrap(); + drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); tokio::time::timeout(Duration::from_secs(3), daemon.ingress_barrier()) .await @@ -613,12 +608,12 @@ mod tests { assert!(daemon.capture_event(next).await.unwrap()); } { - let state = daemon.startup_buffer.lock(); - let buffer = state.as_ref().unwrap(); + let state = daemon.capture_buffer.lock(); + let buffer = &state; assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); assert_eq!(buffer.events.front().unwrap().env.payload["index"], 6); } - drain_startup_buffer(&daemon).await.unwrap(); + drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); daemon.ingress_barrier().await; let source = daemon.translations.lock().values().next().unwrap().clone(); @@ -668,7 +663,7 @@ mod tests { next.payload = serde_json::json!({"index": i}); assert!(daemon.capture_event(next).await.unwrap()); } - let StartupWork::Evicted(first) = daemon.next_startup_work() else { + let CaptureWork::Evicted(first) = daemon.next_capture_work() else { panic!("the oldest events were not evicted"); }; for i in INGRESS_QUEUE_CAPACITY + 2..INGRESS_QUEUE_CAPACITY + 7 { @@ -678,22 +673,20 @@ mod tests { assert!(daemon.capture_event(next).await.unwrap()); } { - let state = daemon.startup_buffer.lock(); - let buffer = state.as_ref().unwrap(); + let state = daemon.capture_buffer.lock(); + let buffer = &state; assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); assert_eq!(buffer.events.front().unwrap().env.payload["index"], 7); } let mut cursors = HashMap::new(); - replay_evicted_startup(&daemon, &mut cursors, &first) - .await - .unwrap(); - let StartupWork::Evicted(second) = daemon.next_startup_work() else { + replay_evicted(&daemon, &mut cursors, &first).await.unwrap(); + let CaptureWork::Evicted(second) = daemon.next_capture_work() else { panic!("new evictions did not advance recovery"); }; - replay_evicted_startup(&daemon, &mut cursors, &second) + replay_evicted(&daemon, &mut cursors, &second) .await .unwrap(); - drain_startup_buffer(&daemon).await.unwrap(); + drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); daemon.ingress_barrier().await; let source = daemon.translations.lock().values().next().unwrap().clone(); @@ -722,7 +715,6 @@ mod tests { let mut opts = debug_serve_options("test", tmp.path()); opts.auth_provider = Some(Arc::new(TestAuth)); let daemon = Daemon::new_with_startup(opts, tmp.path().into(), Some(startup)); - *daemon.startup_buffer.lock() = None; let mut env = event("durable-bound", SessionRoute::default()); env.payload = serde_json::json!({"index":0}); daemon.capture_event(env.clone()).await.unwrap(); @@ -736,7 +728,7 @@ mod tests { unacknowledged.extend(serde_json::to_vec(&env.redacted()).unwrap()); unacknowledged.push(b'\n'); tokio::fs::write(&path, unacknowledged).await.unwrap(); - recover_ingress_overflow(&daemon).await; + drain_capture_buffer(&daemon).await.unwrap(); let source = daemon.translations.lock().values().next().unwrap().clone(); source.barrier().await.unwrap(); let storage = tmp.path().join("pipeline").join(format!( @@ -799,73 +791,7 @@ mod tests { filler.ts_ms = i as i64; daemon.capture_event(filler).await.unwrap(); } - drain_startup_buffer(&daemon).await.unwrap(); - ready.send(()).unwrap(); - daemon.settle_ingress().await; - let route = daemon - .automatic_links - .lock() - .get(&child_key) - .cloned() - .expect("child must be linked"); - let Some(crate::wire::TraceDestination::ParentSpan { components }) = route.destination - else { - panic!("child was not attached to the parent tool"); - }; - let tool_id = crate::ids::span_id( - &crate::ids::session_namespace("opencode", "parent"), - "tool:parent:tool", - ); - assert_eq!(components.span_id.as_deref(), Some(tool_id.as_str())); - drain_all(&daemon).await; - } - - #[tokio::test] - async fn steady_overflow_preserves_parent_child_tool_order() { - use distributed_fixture::{AgentKind, DistributedFixtures, ProcessTree}; - let tmp = tempfile::tempdir().unwrap(); - let (ready, startup) = oneshot::channel(); - let mut opts = debug_serve_options("test", tmp.path()); - opts.auth_provider = Some(Arc::new(TestAuth)); - let daemon = Daemon::new_with_startup(opts, tmp.path().into(), Some(startup)); - // Pause the steady ingress worker while captures saturate its queue. - *daemon.startup_buffer.lock() = None; - let mut fixtures = DistributedFixtures::new(); - let parent = ProcessTree::root(1000); - let child = ProcessTree::child(1002, 1001, &parent); - let mut events = - fixtures.start_turn(AgentKind::OpenCode, "parent", &parent, "delegate", 100); - events.push(fixtures.open_tool( - AgentKind::OpenCode, - "parent", - &parent, - "tool", - "do work", - 102, - )); - let child_events = - fixtures.start_turn(AgentKind::OpenCode, "child", &child, "do work", 103); - let child_key = automatic_link_key(&child_events[0]); - events.extend(child_events); - events.push(fixtures.close_tool( - AgentKind::OpenCode, - "parent", - &parent, - "tool", - "do work", - "done", - 105, - )); - for env in events { - daemon.capture_event(env).await.unwrap(); - } - for i in 0..INGRESS_QUEUE_CAPACITY { - let mut filler = event("filler", SessionRoute::default()); - filler.ts_ms = i as i64; - daemon.capture_event(filler).await.unwrap(); - } - assert!(daemon.ingress_overflow.load(Ordering::Acquire)); - recover_ingress_overflow(&daemon).await; + drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); daemon.settle_ingress().await; let route = daemon @@ -903,11 +829,11 @@ mod tests { assert!(daemon.capture_event(env).await.unwrap()); } { - let state = daemon.startup_buffer.lock(); - let buffer = state.as_ref().unwrap(); + let state = daemon.capture_buffer.lock(); + let buffer = &state; assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); } - drain_startup_buffer(&daemon).await.unwrap(); + drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); daemon.ingress_barrier().await; let sources: Vec<_> = daemon.translations.lock().values().cloned().collect(); @@ -949,25 +875,23 @@ mod tests { env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); assert!(daemon.capture_event(env.clone()).await.unwrap()); } - let StartupWork::Evicted(first) = daemon.next_startup_work() else { + let CaptureWork::Evicted(first) = daemon.next_capture_work() else { panic!("missing first overflow") }; let mut cursors = HashMap::new(); - replay_evicted_startup(&daemon, &mut cursors, &first) - .await - .unwrap(); + replay_evicted(&daemon, &mut cursors, &first).await.unwrap(); for i in INGRESS_QUEUE_CAPACITY + 2..INGRESS_QUEUE_CAPACITY + 4 { env.ts_ms = i as i64; env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); assert!(daemon.capture_event(env.clone()).await.unwrap()); } - let StartupWork::Evicted(second) = daemon.next_startup_work() else { + let CaptureWork::Evicted(second) = daemon.next_capture_work() else { panic!("missing second overflow") }; - replay_evicted_startup(&daemon, &mut cursors, &second) + replay_evicted(&daemon, &mut cursors, &second) .await .unwrap(); - drain_startup_buffer(&daemon).await.unwrap(); + drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); daemon.ingress_barrier().await; let source = daemon.translations.lock().values().next().unwrap().clone(); diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index e5a32ab8..b9a5948d 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -132,15 +132,10 @@ pub struct Daemon { correlation_locks: Mutex>>>, correlation_changed: Arc, ingress_tx: mpsc::Sender, - ingress_overflow: AtomicBool, - ingress_dispatched: Mutex>, - /// Historical prefixes already considered by startup recovery. Capture - /// ordinals in those prefixes belong to an earlier daemon incarnation. - ingress_history: Mutex>, /// Earliest live append during startup, keyed by source journal path. /// Recovery must not absorb queued live events into its historical head. startup_cutoffs: Mutex>>, - startup_buffer: Mutex>, + capture_buffer: Mutex, capture_sequence: AtomicU64, started: Instant, last_activity: Mutex, @@ -166,7 +161,6 @@ impl Daemon { ) -> Arc { let (ingress_tx, ingress_rx) = mpsc::channel(INGRESS_QUEUE_CAPACITY); let startup_cutoffs = startup.as_ref().map(|_| HashMap::new()); - let startup_buffer = startup.as_ref().map(|_| StartupBuffer::new()); let daemon = Arc::new(Daemon { version: opts.version, data_dir, @@ -192,11 +186,8 @@ impl Daemon { correlation_locks: Mutex::new(HashMap::new()), correlation_changed: Arc::new(Notify::new()), ingress_tx, - ingress_overflow: AtomicBool::new(false), - ingress_dispatched: Mutex::new(HashMap::new()), - ingress_history: Mutex::new(HashMap::new()), startup_cutoffs: Mutex::new(startup_cutoffs), - startup_buffer: Mutex::new(startup_buffer), + capture_buffer: Mutex::new(CaptureBuffer::new()), capture_sequence: AtomicU64::new(0), started: Instant::now(), last_activity: Mutex::new(Instant::now()), @@ -283,15 +274,13 @@ pub async fn run(args: ServeArgs, opts: ServeOptions) -> anyhow::Result<()> { .await; restore_pending_sessions(&recovery_daemon).await; let candidates = discover_unprocessed_journals(&recovery_daemon).await; - if let Some(history) = recovery_daemon.startup_cutoffs.lock().take() { - recovery_daemon.ingress_history.lock().extend(history); - } + *recovery_daemon.startup_cutoffs.lock() = None; for event in candidates { dispatch_ingress_event(&recovery_daemon, event).await; } recovery_daemon.settle_session_actors().await; let _ = retry_pending_sessions(&recovery_daemon).await; - if let Err(error) = drain_startup_buffer(&recovery_daemon).await { + if let Err(error) = drain_capture_buffer(&recovery_daemon).await { tracing::error!(%error, "startup ingress recovery failed"); recovery_daemon.trigger_shutdown(); return; diff --git a/bt-daemon/src/server/recovery.rs b/bt-daemon/src/server/recovery.rs index 75e669b3..2dc4e9e8 100644 --- a/bt-daemon/src/server/recovery.rs +++ b/bt-daemon/src/server/recovery.rs @@ -300,10 +300,7 @@ pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec

) { for event in discover_unprocessed_journals(daemon).await { - let _ = daemon - .ingress_tx - .send(IngressMsg::Event(Box::new(event))) - .await; + dispatch_ingress_event(daemon, event).await; } let (reply, _ignored) = oneshot::channel(); let _ = daemon.ingress_tx.send(IngressMsg::Barrier(reply)).await; @@ -391,7 +388,9 @@ mod tests { assert_eq!(candidates.len(), 1); assert_eq!(candidates[0].env.event, "after"); assert_eq!(candidates[0].journal_through, tail); - super::super::ingress::recover_ingress_overflow(&daemon).await; + for event in candidates { + dispatch_ingress_event(&daemon, event).await; + } let source = daemon.translations.lock().values().next().unwrap().clone(); source.barrier().await.unwrap(); assert!(source.error.lock().is_none()); diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index 2d1b0ec7..1150882b 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -250,9 +250,6 @@ impl Daemon { } } self.dispatch_locks.lock().remove(&storage_key); - self.ingress_dispatched.lock().retain(|candidate, _| { - candidate.source != key.source || candidate.session_id != key.session_id - }); } tracing::info!(session_id = %key.session_id, "session retired"); } @@ -288,13 +285,6 @@ impl Daemon { let path = journal::ensure_source_journal(&self.data_dir, source, session_id).await?; let writer = JournalWriter::open_path(&path).await?; - let mut history = self.ingress_history.lock(); - if history - .get(&path) - .is_some_and(|before| *before > writer.position()) - { - history.remove(&path); - } Ok::<_, anyhow::Error>(Arc::new(tokio::sync::Mutex::new(writer))) }) .await?; From 989699e75d9282817ea4a39169d2de0045fa33b4 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 3 Nov 2026 01:42:22 +0800 Subject: [PATCH 14/19] Remove duplicate replay state and share producer initialization --- bt-daemon/src/journal.rs | 4 + bt-daemon/src/journal/codec.rs | 43 +-- bt-daemon/src/pipeline/translation.rs | 304 ++++++------------ bt-daemon/src/server/ingress.rs | 113 +++---- .../tests/support/translation_streams.rs | 20 ++ bt-daemon/tests/translator_checkpoints.rs | 68 +--- 6 files changed, 187 insertions(+), 365 deletions(-) diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index e2d3d475..44905f46 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -428,6 +428,10 @@ pub struct JournalReader { } impl JournalReader { + pub(crate) fn position(&self) -> u64 { + self.position + } + #[cfg(test)] pub(crate) fn decoder_context(&self) -> &DecoderContext { &self.decoder diff --git a/bt-daemon/src/journal/codec.rs b/bt-daemon/src/journal/codec.rs index 9892db6e..0c04805f 100644 --- a/bt-daemon/src/journal/codec.rs +++ b/bt-daemon/src/journal/codec.rs @@ -2,7 +2,7 @@ //! kinds may reference their previous payload, with a bounded per-session cache. use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::{collections::VecDeque, io::Write, sync::Arc}; +use std::{collections::VecDeque, sync::Arc}; // These bounds are part of the v1 decoding contract. const MIN_BYTES: usize = 4 * 1024; @@ -28,45 +28,10 @@ pub(super) struct Delta { insert: String, } -/// Stop serialization once the cache budget is exceeded. Large events remain -/// complete raw records without allocating another unbounded payload copy. -struct LimitedJson { - bytes: Vec, - limit: usize, - exceeded: bool, -} -impl Write for LimitedJson { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - if bytes.len() > self.limit.saturating_sub(self.bytes.len()) { - self.exceeded = true; - return Err(std::io::Error::other("journal codec cache budget exceeded")); - } - self.bytes.extend_from_slice(bytes); - Ok(bytes.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - pub(super) fn cache_json(event: &str, payload: &Value) -> anyhow::Result>> { - if event.len() >= MAX_BYTES { - return Ok(None); - } - let mut output = LimitedJson { - bytes: Vec::new(), - limit: MAX_BYTES - event.len(), - exceeded: false, - }; - let result = serde_json::to_writer(&mut output, payload); - if output.exceeded { - return Ok(None); - } - result?; - if output.bytes.len() < MIN_BYTES { - return Ok(None); - } - Ok(Some(String::from_utf8(output.bytes)?.into())) + // Oversized JSON is dropped before the raw record is serialized. The + // limit bounds retained decoder state; it never truncates event data. + Ok(cache_serialized(event, &serde_json::to_string(payload)?)) } /// Delta offsets refer to the exact stored JSON, including key order and diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 503a4fb2..e1e1117c 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -28,7 +28,6 @@ const CHECKPOINT_BYTES: u64 = 1024 * 1024; #[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct RecoveryContext { - format: u32, translator: Box, pub decoder: crate::journal::DecoderContext, capture: Option, @@ -48,10 +47,6 @@ impl RecoveryContext { ); let decoder = zstd::stream::read::Decoder::new(checkpoint.state.as_slice())?; let context: Self = serde_json::from_reader(std::io::BufReader::new(decoder))?; - anyhow::ensure!( - context.format == 1, - "unsupported translation context format" - ); Ok(context) } } @@ -115,51 +110,12 @@ pub(crate) struct TranslationOptions { impl Translation { pub async fn spawn(options: TranslationOptions) -> anyhow::Result> { - let path = options.data_dir.join("pipeline").join(format!( - "{}.sqlite", - crate::ids::session_storage_id(&options.source, &options.session_id) - )); - let journal = SpanJournal::open(&path).await?; - let committed = journal.translated_through().await?; - let progress = Arc::new(Progress { - requested: AtomicU64::new(committed), - committed: AtomicU64::new(committed), - }); - let recovery = Arc::new(Mutex::new(None)); - let error = Arc::new(Mutex::new(None)); + let actor = Producer::open(options).await?; + let journal = actor.journal.clone(); + let progress = actor.progress.clone(); + let error = actor.error.clone(); + let recovery = actor.recovery.clone(); let (tx, rx) = mpsc::channel(64); - let state = match journal.source_failure().await? { - Some(raw) => State::Recovering(serde_json::from_str(&raw).unwrap_or_else(|_| { - Failure::revision(Boundary::Journal, committed + 1, raw, &options.version) - })), - None => State::Processing, - }; - let actor = Producer { - options, - progress: progress.clone(), - journal: journal.clone(), - error: error.clone(), - recovery: recovery.clone(), - translator: None, - reader: None, - decoder: Default::default(), - observers: Vec::new(), - origins: WeakOrigins::default(), - roots: Roots::default(), - capture: None, - pending_snapshots: HashSet::new(), - bounds: HashMap::new(), - templates: HashMap::new(), - position: 0, - committed, - checkpoint_position: 0, - checkpoint_events: 0, - checkpoint_work_bytes: 0, - checkpoint_safe: true, - state, - publication: None, - phase: Boundary::Journal, - }; let task = tokio::spawn(actor.run(rx)); Ok(Arc::new(Self { tx, @@ -308,6 +264,53 @@ struct Producer { } impl Producer { + async fn open(options: TranslationOptions) -> anyhow::Result { + let path = options.data_dir.join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&options.source, &options.session_id) + )); + let journal = SpanJournal::open(&path).await?; + let committed = journal.translated_through().await?; + let progress = Arc::new(Progress { + requested: AtomicU64::new(committed), + committed: AtomicU64::new(committed), + }); + let recovery = Arc::new(Mutex::new(None)); + let error = Arc::new(Mutex::new(None)); + let state = match journal.source_failure().await? { + Some(raw) => State::Recovering(serde_json::from_str(&raw).unwrap_or_else(|_| { + Failure::revision(Boundary::Journal, committed + 1, raw, &options.version) + })), + None => State::Processing, + }; + Ok(Self { + options, + progress, + journal, + error, + recovery, + translator: None, + reader: None, + decoder: Default::default(), + observers: Vec::new(), + origins: WeakOrigins::default(), + roots: Roots::default(), + capture: None, + pending_snapshots: HashSet::new(), + bounds: HashMap::new(), + templates: HashMap::new(), + position: 0, + committed, + checkpoint_position: 0, + checkpoint_events: 0, + checkpoint_work_bytes: 0, + checkpoint_safe: true, + state, + publication: None, + phase: Boundary::Journal, + }) + } + /// Translators perform synchronous native transcript I/O and reduction. /// Keep both replay and live reduction off the runtime serving hook IPC. async fn translate( @@ -504,7 +507,6 @@ impl Producer { return Ok(None); }; let state = RecoveryContext { - format: 1, translator: serde_json::value::RawValue::from_string(String::from_utf8( state, )?)?, @@ -1089,24 +1091,6 @@ mod tests { #[tokio::test] async fn native_source_output_matches_live_overflow_and_cold_recovery() { - fn comparable(ops: Vec<(u64, SpanOp)>) -> Vec { - let mut rows = ops - .into_iter() - .map(|(_, op)| { - let (SpanOp::Insert(row) | SpanOp::Merge(row)) = &op; - serde_json::to_string(&( - &op, - &row.turn_span_id, - row.is_turn, - &row.late_merge_key, - &row.late_merge_version, - )) - .unwrap() - }) - .collect::>(); - rows.sort(); - rows - } let mut covered = Vec::new(); for (source, mut events, _fixtures) in streams::native_streams() { covered.push(source.clone()); @@ -1164,7 +1148,16 @@ mod tests { .await .unwrap_or_else(|error| panic!("{source} {mode}: {error}")); translation.control(ControlKind::Finalize).await.unwrap(); - let rows = comparable(translation.journal.read_after(0, 10000).await.unwrap()); + let rows = streams::comparable( + translation + .journal + .read_after(0, 10000) + .await + .unwrap() + .into_iter() + .map(|(_, op)| op) + .collect(), + ); assert!(!rows.is_empty(), "{source} produced no observations"); if let Some(expected) = &expected { assert_eq!(&rows, expected, "{source} {mode}"); @@ -1395,38 +1388,12 @@ mod tests { let entered = Arc::new(Notify::new()); let finished = Arc::new(std::sync::atomic::AtomicBool::new(false)); let (release, receiver) = std::sync::mpsc::channel(); - let mut producer = Producer { - options, - journal: SpanJournal::open(&tmp.path().join("work.sqlite")) - .await - .unwrap(), - progress: Arc::default(), - error: Arc::default(), - recovery: Arc::default(), - translator: Some(Box::new(BlockedReplay { - entered: entered.clone(), - finished: finished.clone(), - release: receiver, - })), - reader: None, - decoder: Default::default(), - observers: Vec::new(), - origins: WeakOrigins::default(), - roots: Roots::default(), - capture: None, - pending_snapshots: HashSet::new(), - bounds: HashMap::new(), - templates: HashMap::new(), - position: 0, - committed: 0, - checkpoint_position: 0, - checkpoint_events: 0, - checkpoint_work_bytes: 0, - checkpoint_safe: true, - state: State::Processing, - publication: None, - phase: Boundary::Journal, - }; + let mut producer = Producer::open(options).await.unwrap(); + producer.translator = Some(Box::new(BlockedReplay { + entered: entered.clone(), + finished: finished.clone(), + release: receiver, + })); let started = std::time::Instant::now(); let processing = tokio::spawn(async move { let result = producer.advance(through).await; @@ -1477,40 +1444,16 @@ mod tests { #[tokio::test] async fn slow_checkpoint_encoding_does_not_block_journal_capture_on_the_runtime() { - let (tmp, path, writer) = source_journal("debug").await; + let (tmp, _, writer) = source_journal("debug").await; let entered = Arc::new(Notify::new()); let (release, resume) = std::sync::mpsc::channel(); - let mut producer = Producer { - options: resume_options(tmp.path(), "debug", writer.clone()), - journal: SpanJournal::open(&tmp.path().join("checkpoint.sqlite")) - .await - .unwrap(), - progress: Arc::default(), - error: Arc::default(), - recovery: Arc::default(), - translator: Some(Box::new(BlockedCheckpoint { - entered: entered.clone(), - release: resume, - })), - reader: JournalReader::open(&path, 0).await.unwrap(), - decoder: Default::default(), - observers: Vec::new(), - origins: WeakOrigins::default(), - roots: Roots::default(), - capture: None, - pending_snapshots: HashSet::new(), - bounds: HashMap::new(), - templates: HashMap::new(), - position: 0, - committed: 0, - checkpoint_position: 0, - checkpoint_events: 0, - checkpoint_work_bytes: 0, - checkpoint_safe: true, - state: State::Processing, - publication: None, - phase: Boundary::Journal, - }; + let mut producer = Producer::open(resume_options(tmp.path(), "debug", writer.clone())) + .await + .unwrap(); + producer.translator = Some(Box::new(BlockedCheckpoint { + entered: entered.clone(), + release: resume, + })); let checkpoint = tokio::spawn(async move { producer.checkpoint_context(0).await }); entered.notified().await; let env = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"during-checkpoint","ts_ms":1,"payload":{}})).unwrap(); @@ -1961,72 +1904,39 @@ mod tests { } #[tokio::test] async fn failed_append_and_commit_retain_the_built_chunk() { - let tmp = tempfile::tempdir().unwrap(); - let path = tmp.path().join("spans.sqlite"); - let journal = SpanJournal::open(&path).await.unwrap(); - let db = rusqlite::Connection::open(&path).unwrap(); - let event_path = tmp.path().join("events.ndjson"); + let (tmp, _, writer) = source_journal("debug").await; + let mut producer = Producer::open(resume_options(tmp.path(), "debug", writer)) + .await + .unwrap(); + let journal = producer.journal.clone(); + let path = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id("debug", "checkpoint") + )); + let db = rusqlite::Connection::open(path).unwrap(); let drains = Arc::new(AtomicU64::new(0)); - let mut producer = Producer { - options: TranslationOptions { - source: "debug".into(), - session_id: "test".into(), - namespace: "test".into(), - version: "test".into(), - data_dir: tmp.path().into(), - events: Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&event_path).await.unwrap(), - )), - event_path, - input: SourceInput::default(), - registry: Arc::new(Registry::default_agents()), - correlation: Arc::new(crate::correlation::CorrelationRegistry::default()), - correlation_changed: Arc::new(Notify::new()), - }, - progress: Arc::new(Progress::default()), - journal: journal.clone(), - error: Arc::new(Mutex::new(None)), - recovery: Arc::new(Mutex::new(None)), - translator: Some(Box::new(CountedTranslation(drains.clone()))), - reader: None, - decoder: Default::default(), - observers: Vec::new(), - origins: WeakOrigins::default(), - roots: Roots::default(), + producer.translator = Some(Box::new(CountedTranslation(drains.clone()))); + producer.publication = Some(Publication { + input: 10, + decoded_bytes: 0, + ordinal: 0, + observation: true, capture: None, - pending_snapshots: HashSet::new(), - bounds: HashMap::new(), - templates: HashMap::new(), - position: 0, - committed: 0, - checkpoint_position: 0, - checkpoint_events: 0, - checkpoint_work_bytes: 0, - checkpoint_safe: true, - state: State::Processing, - phase: Boundary::Journal, - publication: Some(Publication { - input: 10, - decoded_bytes: 0, - ordinal: 0, - observation: true, - capture: None, - ops: Some(vec![ - SpanOp::Insert(SpanRow { - span_id: "unused-root".into(), - metadata: Some(serde_json::json!({"_bt_defer_root": true})), - ..Default::default() - }), - SpanOp::Insert(SpanRow { - span_id: "built".into(), - ..Default::default() - }), - ]), - begun: false, - prepared: false, - complete: false, - }), - }; + ops: Some(vec![ + SpanOp::Insert(SpanRow { + span_id: "unused-root".into(), + metadata: Some(serde_json::json!({"_bt_defer_root": true})), + ..Default::default() + }), + SpanOp::Insert(SpanRow { + span_id: "built".into(), + ..Default::default() + }), + ]), + begun: false, + prepared: false, + complete: false, + }); db.execute_batch("CREATE TRIGGER fail_append BEFORE INSERT ON ops BEGIN SELECT RAISE(FAIL,'append'); END;").unwrap(); assert!(producer.publish().await.is_err()); assert_eq!(drains.load(Ordering::Relaxed), 0); diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 8088b694..33518ac3 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -76,84 +76,57 @@ pub(super) async fn drain_capture_buffer(daemon: &Arc) -> anyhow::Result } } -pub(super) struct ReplayCursor { - reader: journal::JournalReader, - position: u64, -} - -/// Advance each evicted journal range from its previous position, retaining -/// the reader's payload decoder context between advances. Older records are decoded -/// only to initialize that context; they are not dispatched again. +/// Resume frozen evicted ranges, merging one lookahead per source in capture +/// order. Retain readers between advances so another eviction can extend them. pub(super) async fn replay_evicted( daemon: &Arc, - cursors: &mut HashMap, + cursors: &mut HashMap, bounds: &HashMap, ) -> anyhow::Result<()> { + let mut ready = Vec::new(); for (path, range) in bounds { + let mut reader = match cursors.remove(path) { + Some(reader) => reader, + None => match resume_discovery(daemon, path, range.from).await { + Ok(Some((reader, _, _))) => reader, + _ => journal::JournalReader::open(path, range.from) + .await? + .ok_or_else(|| anyhow::anyhow!("evicted capture journal disappeared"))?, + }, + }; anyhow::ensure!( - journal::JournalReader::recorded_len(path).await >= range.through, - "evicted capture journal is truncated" - ); - if !cursors.contains_key(path) { - let (reader, position) = match resume_discovery(daemon, path, range.from).await { - Ok(Some((reader, position, _))) => (reader, position), - _ => ( - journal::JournalReader::open(path, range.from) - .await? - .ok_or_else(|| anyhow::anyhow!("evicted capture journal disappeared"))?, - 0, - ), - }; - cursors.insert(path.clone(), ReplayCursor { reader, position }); - } - let cursor = cursors - .get_mut(path) - .ok_or_else(|| anyhow::anyhow!("recovery cursor missing"))?; - anyhow::ensure!( - cursor.position <= range.from, + reader.position() <= range.from, "evicted capture ranges overlap" ); - cursor.reader.extend(range.from); - while cursor.position < range.from { - let entry = - cursor.reader.next_record().await?.ok_or_else(|| { - anyhow::anyhow!("evicted capture range begins inside a record") - })?; - cursor.position = entry.through; - } + // Decode only to reconstruct the payload cache across a gap whose + // events were already dispatched from memory. + reader.extend(range.from); + while reader.next_record().await?.is_some() {} anyhow::ensure!( - cursor.position == range.from, + reader.position() == range.from, "invalid evicted capture boundary" ); - cursor.reader.extend(range.through); - } - // Keep one lookahead per source. The sequence is persisted in the same - // NDJSON append as its event, so capture needs no secondary write queue. - let mut ready = Vec::new(); - for (path, range) in bounds { - let cursor = cursors - .get_mut(path) - .ok_or_else(|| anyhow::anyhow!("recovery cursor missing"))?; - if let Some((sequence, event)) = next_evicted_event(daemon, cursor, range.through).await? { - ready.push((sequence, path.clone(), event)); + reader.extend(range.through); + if let Some((sequence, event)) = + next_evicted_event(daemon, &mut reader, range.through).await? + { + ready.push((sequence, event, path.clone(), reader, range.through)); + } else { + cursors.insert(path.clone(), reader); } } - while !ready.is_empty() { - let index = ready - .iter() - .enumerate() - .min_by_key(|(_, (sequence, _, _))| *sequence) - .map(|(index, _)| index) - .ok_or_else(|| anyhow::anyhow!("recovery lookahead missing"))?; - let (_, path, event) = ready.swap_remove(index); + while let Some(index) = ready + .iter() + .enumerate() + .min_by_key(|(_, item)| item.0) + .map(|(index, _)| index) + { + let (_, event, path, mut reader, through) = ready.swap_remove(index); dispatch_ingress_event(daemon, event).await; - let cursor = cursors - .get_mut(&path) - .ok_or_else(|| anyhow::anyhow!("recovery cursor missing"))?; - if let Some((sequence, event)) = - next_evicted_event(daemon, cursor, bounds[&path].through).await? - { - ready.push((sequence, path, event)); + if let Some((sequence, event)) = next_evicted_event(daemon, &mut reader, through).await? { + ready.push((sequence, event, path, reader, through)); + } else { + cursors.insert(path, reader); } } Ok(()) @@ -161,18 +134,16 @@ pub(super) async fn replay_evicted( async fn next_evicted_event( daemon: &Daemon, - cursor: &mut ReplayCursor, + reader: &mut journal::JournalReader, through: u64, ) -> anyhow::Result> { - while cursor.position < through { - let before = cursor.position; - let entry = cursor - .reader + while reader.position() < through { + let before = reader.position(); + let entry = reader .next_record() .await? .ok_or_else(|| anyhow::anyhow!("replay range ends inside a record"))?; anyhow::ensure!(entry.through <= through, "invalid replay end boundary"); - cursor.position = entry.through; if let journal::JournalRecord::Event(redacted) = entry.record { let mut env = journal::envelope_from_redacted(redacted); let Some(source) = daemon.translators.canonical_source(&env.source) else { @@ -192,7 +163,7 @@ async fn next_evicted_event( ))); } } - anyhow::ensure!(cursor.position == through, "invalid replay end boundary"); + anyhow::ensure!(reader.position() == through, "invalid replay end boundary"); Ok(None) } diff --git a/bt-daemon/tests/support/translation_streams.rs b/bt-daemon/tests/support/translation_streams.rs index 00057179..a04a6119 100644 --- a/bt-daemon/tests/support/translation_streams.rs +++ b/bt-daemon/tests/support/translation_streams.rs @@ -157,3 +157,23 @@ pub fn native_streams() -> Vec { )); cases } + +// Include reducer-only correlation fields omitted by public SpanOp JSON. +pub fn comparable(ops: Vec) -> Vec { + let mut rows = ops + .into_iter() + .map(|op| { + let (bt_daemon::SpanOp::Insert(row) | bt_daemon::SpanOp::Merge(row)) = &op; + serde_json::to_string(&( + &op, + &row.turn_span_id, + row.is_turn, + &row.late_merge_key, + &row.late_merge_version, + )) + .unwrap() + }) + .collect::>(); + rows.sort(); + rows +} diff --git a/bt-daemon/tests/translator_checkpoints.rs b/bt-daemon/tests/translator_checkpoints.rs index fd937b10..34865d0b 100644 --- a/bt-daemon/tests/translator_checkpoints.rs +++ b/bt-daemon/tests/translator_checkpoints.rs @@ -15,24 +15,6 @@ fn collect(translator: &mut dyn AgentTranslator, env: &Envelope, ctx: &SessionCt } ops } -fn comparable(ops: Vec) -> Vec { - let mut values = ops - .into_iter() - .map(|op| { - let (SpanOp::Insert(row) | SpanOp::Merge(row)) = &op; - serde_json::to_string(&( - &op, - &row.turn_span_id, - row.is_turn, - &row.late_merge_key, - &row.late_merge_version, - )) - .unwrap() - }) - .collect::>(); - values.sort(); - values -} fn assert_stream(source: &str, events: Vec) { let registry = Registry::default_agents(); let session = events @@ -56,56 +38,26 @@ fn assert_stream(source: &str, events: Vec) { restored.restore_state(&state).unwrap(); for env in &events[split..] { assert_eq!( - comparable(collect(original.as_mut(), env, &ctx)), - comparable(collect(restored.as_mut(), env, &ctx)), + streams::comparable(collect(original.as_mut(), env, &ctx)), + streams::comparable(collect(restored.as_mut(), env, &ctx)), "{source} split {split} event {}", env.event ); } assert_eq!( - comparable(original.finalize(&ctx).unwrap()), - comparable(restored.finalize(&ctx).unwrap()), + streams::comparable(original.finalize(&ctx).unwrap()), + streams::comparable(restored.finalize(&ctx).unwrap()), "{source} split {split} finalization" ); } } #[test] -fn checkpoints_preserve_native_turns_tools_transcripts_and_later_turn_ids() { - for (source, events, _fixtures) in streams::native_streams() { - if ["claude-code", "codex", "pi", "opencode"].contains(&source.as_str()) { - assert_stream(&source, events); - } - } -} -#[test] -fn muse_checkpoints_preserve_open_operations_and_cumulative_usage() { - for (source, events, _fixtures) in streams::native_streams() { - if source == "muse" { - assert_stream(&source, events); - } - } -} -#[test] -fn cursor_checkpoints_preserve_open_tools_and_internal_span_fields() { +fn every_translator_preserves_native_state_at_every_event_boundary() { + let mut covered = Vec::new(); for (source, events, _fixtures) in streams::native_streams() { - if source == "cursor" { - assert_stream(&source, events); - } - } -} -#[test] -fn antigravity_checkpoints_preserve_invocations_across_native_boundaries() { - for (source, events, _fixtures) in streams::native_streams() { - if source == "antigravity" { - assert_stream(&source, events); - } - } -} -#[test] -fn grok_checkpoints_preserve_transcript_cursors_and_pending_model_work() { - for (source, events, _fixtures) in streams::native_streams() { - if source == "grok" { - assert_stream(&source, events); - } + covered.push(source.clone()); + assert_stream(&source, events); } + covered.sort(); + assert_eq!(covered, Registry::default_agents().sources()); } From 9c5035ec7fc2358a28715372df07bd705ecb22c0 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 3 Nov 2026 02:26:40 +0800 Subject: [PATCH 15/19] Simplify history storage and consolidate overlapping recovery tests --- bt-daemon/src/journal.rs | 246 +---- bt-daemon/src/journal/codec.rs | 276 ++--- bt-daemon/src/pipeline/input.rs | 52 +- bt-daemon/src/pipeline/journal.rs | 35 +- bt-daemon/src/pipeline/translation.rs | 967 ++++++------------ bt-daemon/src/server/ingress.rs | 10 +- bt-daemon/src/server/maintenance.rs | 469 +++------ bt-daemon/src/server/mod.rs | 2 +- bt-daemon/src/server/recovery.rs | 74 +- bt-daemon/src/translate/pi.rs | 34 - bt-daemon/src/transport.rs | 11 +- .../tests/support/translation_streams.rs | 191 +--- bt-daemon/tests/translator_checkpoints.rs | 63 -- 13 files changed, 688 insertions(+), 1742 deletions(-) delete mode 100644 bt-daemon/tests/translator_checkpoints.rs diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index 44905f46..25866641 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -13,7 +13,7 @@ mod codec; use crate::pipeline::input::SourcePosition; pub use crate::pipeline::input::{ControlKind, PipelineControl}; -pub(crate) use codec::DecoderContext; +pub(crate) use codec::CodecState; use crate::wire::{BackendAuth, Envelope, RedactedEnvelope, SessionRoute}; use serde::{Deserialize, Serialize}; @@ -226,7 +226,7 @@ pub async fn gc_old_managed_runs(data_dir: &Path, max_age: std::time::Duration) pub struct JournalWriter { file: tokio::fs::File, position: u64, - payloads: codec::PayloadCache, + codec: CodecState, failed_write: bool, } @@ -275,7 +275,7 @@ impl JournalWriter { Ok(Self { file, position, - payloads: codec::PayloadCache::default(), + codec: CodecState::default(), failed_write: false, }) } @@ -333,7 +333,7 @@ impl JournalWriter { _bt_capture_sequence: Option, } let mut redacted = env.redacted(); - let (json, delta) = self.payloads.encode(&env.event, &redacted.payload)?; + let (json, delta) = self.codec.encode(&redacted.payload)?; let record_type = if let Some(delta) = delta { redacted.payload = serde_json::to_value(delta)?; codec::DELTA_RECORD @@ -347,8 +347,7 @@ impl JournalWriter { })?; line.push(b'\n'); let before = self.append_bytes(&line).await?; - self.payloads - .record(&env.event, self.position, json.clone()); + self.codec.record(self.position, json.clone()); Ok(SourcePosition { before, through: self.position, @@ -424,7 +423,7 @@ pub struct JournalReader { path: PathBuf, line_no: usize, position: u64, - decoder: DecoderContext, + decoder: CodecState, } impl JournalReader { @@ -432,13 +431,8 @@ impl JournalReader { self.position } - #[cfg(test)] - pub(crate) fn decoder_context(&self) -> &DecoderContext { - &self.decoder - } - /// The source consumer owns decoding state between recovery reads. - pub(crate) fn swap_decoder_context(&mut self, context: &mut DecoderContext) { + pub(crate) fn swap_decoder_context(&mut self, context: &mut CodecState) { std::mem::swap(&mut self.decoder, context); } @@ -448,8 +442,9 @@ impl JournalReader { path: &Path, position: u64, through: u64, - decoder_context: DecoderContext, + decoder_context: CodecState, ) -> anyhow::Result { + decoder_context.validate()?; let mut file = tokio::fs::File::open(path).await?; anyhow::ensure!( position <= through && through <= file.metadata().await?.len(), @@ -489,7 +484,7 @@ impl JournalReader { path: path.to_path_buf(), line_no: 0, position: 0, - decoder: DecoderContext::default(), + decoder: CodecState::default(), })) } @@ -583,7 +578,7 @@ fn decode_record( path: &Path, line_no: usize, position: u64, - decoder: &mut DecoderContext, + decoder: &mut CodecState, ) -> anyhow::Result> { let line = std::str::from_utf8(bytes) .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))? @@ -625,7 +620,7 @@ fn decode_record( Some("event") | Some(codec::DELTA_RECORD) => { decoder.legacy_pi.clear(); let json = if record_type.as_deref() == Some(codec::DELTA_RECORD) { - let (payload, json) = decoder.payloads.decode(&event.event, event.payload)?; + let (payload, json) = decoder.decode(event.payload)?; decoded_bytes = decoded_bytes.max(json.len() as u64); event.payload = payload; Some(json) @@ -636,12 +631,12 @@ fn decode_record( payload: &'a serde_json::value::RawValue, } let stored: StoredPayload<'_> = serde_json::from_str(line)?; - codec::cache_serialized(&event.event, stored.payload.get()) + codec::cache_serialized(stored.payload.get()) }; - decoder.payloads.record(&event.event, position, json); + decoder.record(position, json); } None => { - decoder.payloads = codec::PayloadCache::default(); + decoder.base = None; expand_legacy_pi_payload(&mut event, &mut decoder.legacy_pi).map_err(|error| { anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no) })?; @@ -845,13 +840,17 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let path = tmp.path().join("original-json.ndjson"); // Key order and number formatting may differ across serde feature - // sets/versions. Offsets always refer to the writer's exact JSON. + // sets/versions. Dictionaries use the writer's exact JSON. let raw = format!(r#"{{"z":"{}","a":1e-3}}"#, "x".repeat(4096)); let first = format!("{{\"_bt_record_type\":\"event\",\"source\":\"debug\",\"session_id\":\"bytes\",\"event\":\"native\",\"ts_ms\":1,\"payload\":{raw}}}\n"); - let prefix = raw.find("1e-3").unwrap(); + let mut cache = CodecState::default(); + cache.record(first.len() as u64, Some(raw.clone().into())); + let (_, delta) = cache + .encode(&serde_json::json!({"z":"x".repeat(4096),"a":0.002})) + .unwrap(); let delta = serde_json::json!({ "_bt_record_type":codec::DELTA_RECORD, "source":"debug", "session_id":"bytes", "event":"native", "ts_ms":2, - "payload":{"base":first.len(),"prefix":prefix,"suffix":raw.len()-prefix-4,"insert":"2e-3"} + "payload":delta.unwrap() }); tokio::fs::write(&path, format!("{first}{delta}\n")) .await @@ -868,193 +867,42 @@ mod tests { } #[tokio::test] - async fn failed_append_retry_preserves_the_durable_prefix_and_delta_base() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("retry.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let first = pi_context( - vec![serde_json::json!({"content":"history".repeat(1024)})], - 1, - ); - let through = writer.append(&first).await.unwrap(); - // Model a failed write which left OS-visible bytes but did not finish - // durability or update the codec's acknowledged base. - writer.file.write_all(b"{unfinished").await.unwrap(); - writer.file.flush().await.unwrap(); - writer.failed_write = true; - let mut second = first.clone(); - second.payload["tail"] = "after retry".into(); - let position = writer.append_ordered(&second, None).await.unwrap(); - assert_eq!(position.before, through); - let stored = tokio::fs::read_to_string(&path).await.unwrap(); - assert!(stored.contains(codec::DELTA_RECORD)); - let mut reader = JournalReader::open(&path, position.through) - .await - .unwrap() - .unwrap(); - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - first.payload - ); - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - second.payload - ); - assert!(reader.next_entry().await.unwrap().is_none()); - } - - #[tokio::test] - async fn torn_tail_is_excluded_and_removed_before_new_appends() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("events.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let event = pi_context(vec![serde_json::json!({"content":"complete"})], 1); - let through = writer.append(&event).await.unwrap(); - drop(writer); - let mut stored = tokio::fs::read(&path).await.unwrap(); - stored.extend_from_slice(b"{\"source\":\"pi\",\"payload\":"); - tokio::fs::write(&path, stored).await.unwrap(); - assert_eq!( - JournalReader::recorded_len(&path).await, - through, - "a torn trailing write is outside the recovery snapshot" - ); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - assert_eq!(writer.position(), through); - let tail = writer.append(&event).await.unwrap(); - let mut reader = JournalReader::open(&path, tail).await.unwrap().unwrap(); - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - event.payload - ); - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - event.payload - ); - assert!(reader.next_entry().await.unwrap().is_none()); - } - - #[tokio::test] - async fn small_payloads_preserve_unknown_native_fields_without_deltas() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("events.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let mut event = pi_context( - vec![serde_json::json!({"role":"user","content":"history"})], - 1, - ); - event.payload["unknown_provider_field"] = serde_json::json!({"preserve":true}); - let tail = writer.append(&event).await.unwrap(); - let stored: serde_json::Value = - serde_json::from_str(&tokio::fs::read_to_string(&path).await.unwrap()).unwrap(); - assert_eq!(stored["payload"], event.payload); - let mut reader = JournalReader::open(&path, tail).await.unwrap().unwrap(); - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - event.payload - ); - assert!(reader.decoder_context().legacy_pi.is_empty()); - } - - #[tokio::test] - async fn every_source_deduplicates_repeated_payloads_and_resumes_losslessly() { - for source in crate::translate::Registry::default_agents().sources() { + async fn partial_writes_recover_on_retry_and_reopen_without_losing_payloads() { + for reopen in [false, true] { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("events.ndjson"); let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let mut expected = Vec::new(); - let mut messages = Vec::new(); - let mut raw_bytes = 0; - for index in 0..20 { - messages.push(serde_json::json!({"role":"user","content":format!("{index}:{}", "x".repeat(4096))})); - let mut event = pi_context(messages.clone(), index); - event.source = source.clone(); - raw_bytes += serde_json::to_vec(&event.redacted()).unwrap().len() + 1; - writer.append(&event).await.unwrap(); - expected.push(event); - } - drop(writer); - let stored = tokio::fs::read_to_string(&path).await.unwrap(); - assert!( - stored.len() < raw_bytes / 3, - "{source} should benefit from the shared payload codec" + let first = pi_context( + vec![serde_json::json!({"content":"history".repeat(1024)})], + 1, ); - assert!(stored.contains(codec::DELTA_RECORD)); - let mut reader = JournalReader::open(&path, stored.len() as u64) + let through = writer.append(&first).await.unwrap(); + writer.file.write_all(b"{unfinished").await.unwrap(); + writer.file.flush().await.unwrap(); + assert_eq!(JournalReader::recorded_len(&path).await, through); + if reopen { + writer = JournalWriter::open_path(&path).await.unwrap(); + } else { + writer.failed_write = true; + } + let mut second = first.clone(); + second.payload["tail"] = "after recovery".into(); + let position = writer.append_ordered(&second, None).await.unwrap(); + assert_eq!(position.before, through); + let mut reader = JournalReader::open(&path, position.through) .await .unwrap() .unwrap(); - let mut boundary = 0; - for (index, event) in expected.iter().enumerate() { - let entry = reader.next_record().await.unwrap().unwrap(); - let JournalRecord::Event(replayed) = entry.record else { - panic!("expected event") - }; - assert_eq!(replayed.payload, event.payload, "{source} event {index}"); - boundary = entry.through; - // Restore directly in the middle of a delta chain, with no - // prefix replay and without assumptions about agent schemas. - if index == 9 { - let saved = serde_json::to_vec(reader.decoder_context()).unwrap(); - let context = serde_json::from_slice(&saved).unwrap(); - reader = JournalReader::resume(&path, boundary, stored.len() as u64, context) - .await - .unwrap(); - } + for event in [first, second] { + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + event.payload + ); } - assert_eq!(boundary, stored.len() as u64); + assert!(reader.next_entry().await.unwrap().is_none()); } } - #[tokio::test] - async fn legacy_pi_deltas_recover_and_mix_with_complete_new_events() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("pi.ndjson"); - let first = pi_context( - vec![serde_json::json!({"role":"user","content":"old history"})], - 1, - ); - let mut delta = pi_context(Vec::new(), 2).redacted(); - delta.payload["event"] - .as_object_mut() - .unwrap() - .remove("messages"); - delta.payload["event"][LEGACY_PI_MESSAGES_DELTA] = serde_json::json!({"common_prefix":1,"suffix":[{"role":"assistant","content":"legacy delta"}]}); - let first_line = format!("{}\n", serde_json::to_string(&first.redacted()).unwrap()); - let old_log = format!("{}{}\n", first_line, serde_json::to_string(&delta).unwrap()); - tokio::fs::write(&path, old_log.as_bytes()).await.unwrap(); - let mut prefix = JournalReader::open(&path, first_line.len() as u64) - .await - .unwrap() - .unwrap(); - prefix.next_entry().await.unwrap().unwrap(); - let decoder = prefix.decoder_context().clone(); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let mut native = pi_context( - vec![serde_json::json!({"role":"user","content":"new native snapshot"})], - 3, - ); - // Even an identically named field inside a native payload must be - // preserved. Only untagged old records use the compatibility codec. - native.payload["event"][LEGACY_PI_MESSAGES_DELTA] = - serde_json::json!({"native_field":"preserve"}); - let tail = writer.append(&native).await.unwrap(); - drop(writer); - let mut reader = JournalReader::resume(&path, first_line.len() as u64, tail, decoder) - .await - .unwrap(); - let expanded = reader.next_entry().await.unwrap().unwrap(); - assert_eq!( - expanded.payload["event"]["messages"], - serde_json::json!([{"role":"user","content":"old history"},{"role":"assistant","content":"legacy delta"}]) - ); - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - native.payload - ); - assert!(reader.decoder_context().legacy_pi.is_empty()); - } - #[tokio::test] async fn source_journals_are_distinct_and_migrate_legacy_history() { let temp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/journal/codec.rs b/bt-daemon/src/journal/codec.rs index 0c04805f..cf0b5df6 100644 --- a/bt-daemon/src/journal/codec.rs +++ b/bt-daemon/src/journal/codec.rs @@ -1,204 +1,95 @@ -//! Lossless payload encoding shared by every source journal. Repeated event -//! kinds may reference their previous payload, with a bounded per-session cache. +//! Lossless history compression using the last eligible payload as a zstd +//! dictionary. All sources share this storage codec; live events stay intact. +use base64::{engine::general_purpose::STANDARD, Engine}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::{collections::VecDeque, sync::Arc}; +use std::sync::Arc; -// These bounds are part of the v1 decoding contract. const MIN_BYTES: usize = 4 * 1024; -const MAX_BASES: usize = 8; const MAX_BYTES: usize = 8 * 1024 * 1024; -pub(super) const DELTA_RECORD: &str = "payload_delta_v1"; +pub(super) const DELTA_RECORD: &str = "payload_zstd_v1"; #[derive(Clone, Default, Serialize, Deserialize)] -pub(crate) struct PayloadCache { - entries: VecDeque, +pub(crate) struct CodecState { + pub(crate) legacy_pi: Vec, + pub(super) base: Option, } #[derive(Clone, Serialize, Deserialize)] -struct Base { - event: String, +pub(super) struct Base { through: u64, json: Arc, } #[derive(Serialize, Deserialize)] pub(super) struct Delta { base: u64, - prefix: usize, - suffix: usize, - insert: String, -} - -pub(super) fn cache_json(event: &str, payload: &Value) -> anyhow::Result>> { - // Oversized JSON is dropped before the raw record is serialized. The - // limit bounds retained decoder state; it never truncates event data. - Ok(cache_serialized(event, &serde_json::to_string(payload)?)) + data: String, } -/// Delta offsets refer to the exact stored JSON, including key order and -/// number spelling. Re-serializing a decoded Value is not a portable base. -pub(super) fn cache_serialized(event: &str, json: &str) -> Option> { - if json.len() < MIN_BYTES || json.len() > MAX_BYTES.saturating_sub(event.len()) { - None - } else { - Some(Arc::from(json)) - } +// Cache limits govern retained state, never which event data is journaled. +// Preserve the writer's exact JSON bytes as the compression dictionary. +pub(super) fn cache_serialized(json: &str) -> Option> { + (MIN_BYTES..=MAX_BYTES) + .contains(&json.len()) + .then(|| Arc::from(json)) } -impl PayloadCache { +impl CodecState { pub(super) fn encode( &self, - event: &str, payload: &Value, ) -> anyhow::Result<(Option>, Option)> { - let Some(json) = cache_json(event, payload)? else { + let Some(json) = cache_serialized(&serde_json::to_string(payload)?) else { return Ok((None, None)); }; - let Some(base) = self.entries.iter().find(|base| base.event == event) else { + let Some(base) = &self.base else { return Ok((Some(json), None)); }; - let mut prefix = base - .json - .bytes() - .zip(json.bytes()) - .take_while(|(a, b)| a == b) - .count(); - while !base.json.is_char_boundary(prefix) || !json.is_char_boundary(prefix) { - prefix -= 1; - } - let remaining = (base.json.len() - prefix).min(json.len() - prefix); - let mut suffix = base - .json - .bytes() - .rev() - .zip(json.bytes().rev()) - .take(remaining) - .take_while(|(a, b)| a == b) - .count(); - while !base.json.is_char_boundary(base.json.len() - suffix) - || !json.is_char_boundary(json.len() - suffix) - { - suffix -= 1; - } + let mut compressor = zstd::bulk::Compressor::with_dictionary(1, base.json.as_bytes())?; + compressor.set_parameter(zstd::zstd_safe::CParameter::ChecksumFlag(true))?; let delta = Delta { base: base.through, - prefix, - suffix, - insert: json[prefix..json.len() - suffix].to_owned(), + data: STANDARD.encode(compressor.compress(json.as_bytes())?), }; - // Account for the extra record tag and avoid expanding small payloads. let worthwhile = serde_json::to_vec(&delta)?.len() + 16 < json.len(); Ok((Some(json), worthwhile.then_some(delta))) } - pub(super) fn record(&mut self, event: &str, through: u64, json: Option>) { - self.entries.retain(|base| base.event != event); - let Some(json) = json else { - return; - }; - if json.len() > MAX_BYTES.saturating_sub(event.len()) { - return; + pub(super) fn record(&mut self, through: u64, json: Option>) { + // Small intervening hooks need not discard the last history base. + if let Some(json) = json { + self.base = Some(Base { through, json }); } - while self.entries.len() >= MAX_BASES - || self - .entries - .iter() - .map(|base| base.json.len() + base.event.len()) - .sum::() - + json.len() - + event.len() - > MAX_BYTES - { - self.entries.pop_front(); - } - self.entries.push_back(Base { - event: event.to_owned(), - through, - json, - }); } - pub(super) fn decode(&self, event: &str, encoded: Value) -> anyhow::Result<(Value, Arc)> { + pub(super) fn decode(&self, encoded: Value) -> anyhow::Result<(Value, Arc)> { let delta: Delta = serde_json::from_value(encoded)?; let base = self - .entries - .iter() - .find(|base| base.event == event && base.through == delta.base) - .ok_or_else(|| anyhow::anyhow!("journal payload delta base is unavailable"))?; - let kept = delta - .prefix - .checked_add(delta.suffix) - .ok_or_else(|| anyhow::anyhow!("invalid journal delta bounds"))?; - anyhow::ensure!( - kept <= base.json.len() - && base.json.is_char_boundary(delta.prefix) - && base.json.is_char_boundary(base.json.len() - delta.suffix), - "invalid journal delta bounds" - ); - let budget = MAX_BYTES.saturating_sub(event.len()); + .base + .as_ref() + .filter(|base| base.through == delta.base) + .ok_or_else(|| anyhow::anyhow!("journal dictionary base is unavailable"))?; anyhow::ensure!( - delta.insert.len() <= budget && kept <= budget - delta.insert.len(), - "journal delta exceeds codec budget" + delta.data.len() <= MAX_BYTES, + "compressed payload exceeds codec budget" ); - let mut json = String::with_capacity(kept + delta.insert.len()); - json.push_str(&base.json[..delta.prefix]); - json.push_str(&delta.insert); - json.push_str(&base.json[base.json.len() - delta.suffix..]); + let bytes = zstd::bulk::Decompressor::with_dictionary(base.json.as_bytes())? + .decompress(&STANDARD.decode(delta.data)?, MAX_BYTES)?; + let json = String::from_utf8(bytes)?; Ok((serde_json::from_str(&json)?, json.into())) } - pub(super) fn validate(&self) -> anyhow::Result<()> { + pub(crate) fn validate(&self) -> anyhow::Result<()> { anyhow::ensure!( - self.entries.len() <= MAX_BASES - && self - .entries - .iter() - .map(|base| base.json.len() + base.event.len()) - .sum::() - <= MAX_BYTES, - "journal decoder checkpoint exceeds codec budget" + self.base + .as_ref() + .is_none_or(|base| base.json.len() <= MAX_BYTES), + "journal dictionary exceeds codec budget" ); Ok(()) } -} - -#[derive(Clone, Default, Serialize)] -pub(crate) struct DecoderContext { - pub(crate) legacy_pi: Vec, - pub(super) payloads: PayloadCache, -} -impl DecoderContext { - /// Observe storage metadata independently of the original live payload. - /// This base is saved for recovery; live translation never decodes it. - pub(crate) fn record_live( - &mut self, - event: &str, - position: &crate::pipeline::input::SourcePosition, - ) { + pub(crate) fn record_live(&mut self, position: &crate::pipeline::input::SourcePosition) { self.legacy_pi.clear(); - self.payloads - .record(event, position.through, position.decoder_base.clone()); - } -} -impl<'de> Deserialize<'de> for DecoderContext { - fn deserialize>(deserializer: D) -> Result { - #[derive(Deserialize)] - struct Stored { - legacy_pi: Vec, - payloads: PayloadCache, - } - let Stored { - legacy_pi, - payloads, - } = Stored::deserialize(deserializer)?; - let context = Self { - legacy_pi, - payloads, - }; - context - .payloads - .validate() - .map_err(serde::de::Error::custom)?; - Ok(context) + self.record(position.through, position.decoder_base.clone()); } } @@ -208,69 +99,28 @@ mod tests { use serde_json::json; #[test] - fn unicode_prefix_suffix_deltas_are_lossless_and_reject_missing_bases() { - let first = json!({"data":format!("{}é{}", "é".repeat(1000), "tail".repeat(1000))}); - let next = json!({"data":format!("{}ê{}", "é".repeat(1000), "tail".repeat(1000))}); - let mut cache = PayloadCache::default(); - let (base, delta) = cache.encode("arbitrary-event", &first).unwrap(); + fn dictionary_roundtrip_is_lossless_bounded_and_rejects_invalid_bases() { + let first = json!({"history":"é📚".repeat(1024),"index":1}); + let next = json!({"history":format!("{}more context", "é📚".repeat(1024)),"index":2}); + let mut cache = CodecState::default(); + let (base, delta) = cache.encode(&first).unwrap(); assert!(delta.is_none()); - cache.record("arbitrary-event", 123, base); - let (_, delta) = cache.encode("arbitrary-event", &next).unwrap(); + cache.record(100, base); + let (_, delta) = cache.encode(&next).unwrap(); let encoded = serde_json::to_value(delta.unwrap()).unwrap(); - assert_eq!( - cache.decode("arbitrary-event", encoded.clone()).unwrap().0, - next - ); - assert!(PayloadCache::default() - .decode("arbitrary-event", encoded.clone()) - .is_err()); - let mut invalid = encoded; - invalid["prefix"] = json!(usize::MAX); - assert!(cache.decode("arbitrary-event", invalid).is_err()); - } - - #[test] - fn cache_limits_evict_old_bases_and_preserve_oversized_payloads() { - let mut cache = PayloadCache::default(); - let payload = json!({"data":"x".repeat(MAX_BYTES / 3)}); - for index in 0..MAX_BASES + 2 { - let name = format!("event-{index}"); - let (json, _) = cache.encode(&name, &payload).unwrap(); - cache.record(&name, index as u64, json); - cache.validate().unwrap(); + assert!(encoded.to_string().len() < first.to_string().len() / 3); + assert_eq!(cache.decode(encoded.clone()).unwrap().0, next); + assert!(CodecState::default().decode(encoded.clone()).is_err()); + cache.record(101, cache.base.as_ref().map(|base| base.json.clone())); + assert!(cache.decode(encoded.clone()).is_err()); + cache.record(100, Some("different dictionary".into())); + assert!(cache.decode(encoded).is_err()); + for payload in [ + json!({"small":1}), + json!({"large":"x".repeat(MAX_BYTES + 1)}), + ] { + let (base, delta) = cache.encode(&payload).unwrap(); + assert!(base.is_none() && delta.is_none()); } - assert!(cache.entries.iter().all(|base| base.event != "event-0")); - let (small, delta) = cache.encode("small", &json!({"value":1})).unwrap(); - assert!( - small.is_none() && delta.is_none(), - "small hook payloads should retain the direct serialization path" - ); - let oversized = json!({"data":"x".repeat(MAX_BYTES + 1)}); - let (json, delta) = cache.encode("large", &oversized).unwrap(); - assert!(json.is_none()); - assert!(delta.is_none()); - cache.record("large", 100, json); - cache.record("large", 101, Some("x".repeat(MAX_BYTES + 1).into())); - cache.validate().unwrap(); - assert!(cache.entries.iter().all(|base| base.event != "large")); - } - - #[test] - fn checkpoints_preserve_shared_bases() { - let mut context = DecoderContext::default(); - let payload = json!({"messages":["content".repeat(1000)]}); - let (json, _) = context.payloads.encode("message", &payload).unwrap(); - context.payloads.record("message", 100, json); - let saved = serde_json::to_vec(&context).unwrap(); - let restored: DecoderContext = serde_json::from_slice(&saved).unwrap(); - let (_, delta) = context.payloads.encode("message", &payload).unwrap(); - assert_eq!( - restored - .payloads - .decode("message", serde_json::to_value(delta.unwrap()).unwrap()) - .unwrap() - .0, - payload - ); } } diff --git a/bt-daemon/src/pipeline/input.rs b/bt-daemon/src/pipeline/input.rs index f7548d1e..5bee8698 100644 --- a/bt-daemon/src/pipeline/input.rs +++ b/bt-daemon/src/pipeline/input.rs @@ -117,47 +117,29 @@ mod tests { use super::*; use serde_json::json; - fn event(source: &str, index: usize, bytes: usize) -> Envelope { - serde_json::from_value(json!({ - "source":source, "session_id":"direct", "event":"native", "ts_ms":index, - "payload":{"history":"x".repeat(bytes),"index":index} - })) - .unwrap() - } - - fn position(before: u64, event: &Envelope) -> SourcePosition { - let bytes = serde_json::to_vec(event).unwrap().len() as u64; - SourcePosition { - before, - through: before + bytes, - decoded_bytes: bytes, - decoder_base: None, - } - } - #[test] - fn oversized_events_flow_directly_but_eviction_keeps_the_input_bounded() { + fn oversized_events_flow_alone_and_eviction_exposes_a_recovery_gap() { let input = SourceInput::default(); - let oversized = event("debug", 0, 9 * 1024 * 1024); - let first = position(0, &oversized); - let boundary = first.through; - input.push_event(oversized.clone(), first); - assert!(input.take(0, boundary - 1).is_none()); - let SourceEvent::Event(actual) = input.take(0, boundary).unwrap().event else { + let event: Envelope = serde_json::from_value(json!({"source":"debug","session_id":"direct","event":"native","ts_ms":1,"payload":{"history":"x".repeat(9 * 1024 * 1024)}})).unwrap(); + let position = |before, through| SourcePosition { + before, + through, + decoded_bytes: 9 * 1024 * 1024, + decoder_base: None, + }; + input.push_event(event.clone(), position(0, 1)); + assert!(input.take(0, 0).is_none()); + let SourceEvent::Event(actual) = input.take(0, 1).unwrap().event else { panic!("expected original event"); }; - assert_eq!(actual.payload, oversized.payload); - input.push_event(oversized.clone(), position(boundary, &oversized)); - let gap_end = position(boundary, &oversized).through; - let next = event("debug", 1, 3 * 1024 * 1024); - let next_position = position(gap_end, &next); - let through = next_position.through; - input.push_event(next, next_position); + assert_eq!(actual.payload, event.payload); + input.push_event(event.clone(), position(1, 2)); + input.push_event(event, position(2, 3)); assert!( - input.take(boundary, through).is_none(), - "evicted input requires recovery of the gap" + input.take(1, 3).is_none(), + "eviction requires recovery of the gap" ); - assert!(input.take(gap_end, through).is_some()); + assert!(input.take(2, 3).is_some()); assert!(input.is_empty()); } } diff --git a/bt-daemon/src/pipeline/journal.rs b/bt-daemon/src/pipeline/journal.rs index ef22a041..6c0e237d 100644 --- a/bt-daemon/src/pipeline/journal.rs +++ b/bt-daemon/src/pipeline/journal.rs @@ -1941,18 +1941,35 @@ mod tests { assert!(roots.process(&vec![root]).await.is_empty()); journal.begin(10, true).await.unwrap(); let db = Connection::open(&path).unwrap(); - db.execute_batch("CREATE TRIGGER fail_roots BEFORE INSERT ON root_candidates BEGIN SELECT RAISE(FAIL,'root write'); END;").unwrap(); - assert!(journal - .commit_with_roots(10, roots.pending()) - .await - .is_err()); - assert_eq!(journal.translated_through().await.unwrap(), 0); - assert!(journal.root_candidates().await.unwrap().is_empty()); - db.execute_batch("DROP TRIGGER fail_roots").unwrap(); + let context = TranslationCheckpoint { + input: 10, + version: "test".into(), + state: vec![1, 2], + }; + for table in ["root_candidates", "translation_checkpoint"] { + db.execute_batch(&format!("CREATE TRIGGER fail_state BEFORE INSERT ON {table} BEGIN SELECT RAISE(FAIL,'state write'); END;")).unwrap(); + assert!(journal + .commit_with_context(10, roots.pending(), Some(context.clone())) + .await + .is_err()); + assert_eq!(journal.translated_through().await.unwrap(), 0); + assert!(journal.root_candidates().await.unwrap().is_empty()); + assert!(journal.translation_checkpoint().await.unwrap().is_none()); + db.execute_batch("DROP TRIGGER fail_state").unwrap(); + } journal - .commit_with_roots(10, roots.pending()) + .commit_with_context(10, roots.pending(), Some(context)) .await .unwrap(); + assert_eq!( + journal + .translation_checkpoint() + .await + .unwrap() + .unwrap() + .input, + 10 + ); journal.close().await.unwrap(); let journal = SpanJournal::open(&path).await.unwrap(); diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index e1e1117c..d6dd575e 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -29,7 +29,7 @@ const CHECKPOINT_BYTES: u64 = 1024 * 1024; #[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct RecoveryContext { translator: Box, - pub decoder: crate::journal::DecoderContext, + pub decoder: crate::journal::CodecState, capture: Option, bounds: HashMap, pub templates: HashMap, @@ -47,6 +47,7 @@ impl RecoveryContext { ); let decoder = zstd::stream::read::Decoder::new(checkpoint.state.as_slice())?; let context: Self = serde_json::from_reader(std::io::BufReader::new(decoder))?; + context.decoder.validate()?; Ok(context) } } @@ -94,6 +95,7 @@ pub(crate) struct Translation { pub progress: Arc, } +#[derive(Clone)] pub(crate) struct TranslationOptions { pub source: String, pub session_id: String, @@ -244,7 +246,7 @@ struct Producer { recovery: Arc>>, translator: Option>, reader: Option, - decoder: crate::journal::DecoderContext, + decoder: crate::journal::CodecState, observers: Vec, origins: WeakOrigins, roots: Roots, @@ -255,8 +257,7 @@ struct Producer { position: u64, committed: u64, checkpoint_position: u64, - checkpoint_events: u64, - checkpoint_work_bytes: u64, + checkpoint_work: u64, checkpoint_safe: bool, state: State, publication: Option, @@ -302,8 +303,7 @@ impl Producer { position: 0, committed, checkpoint_position: 0, - checkpoint_events: 0, - checkpoint_work_bytes: 0, + checkpoint_work: 0, checkpoint_safe: true, state, publication: None, @@ -539,8 +539,7 @@ impl Producer { if let Some(checkpoint) = self.checkpoint_context(self.committed).await? { self.journal.save_translation_checkpoint(checkpoint).await?; self.checkpoint_position = self.committed; - self.checkpoint_events = 0; - self.checkpoint_work_bytes = 0; + self.checkpoint_work = 0; } Ok(()) } @@ -642,8 +641,7 @@ impl Producer { self.position = 0; self.decoder = Default::default(); self.checkpoint_position = 0; - self.checkpoint_events = 0; - self.checkpoint_work_bytes = 0; + self.checkpoint_work = 0; self.checkpoint_safe = true; if let Some(checkpoint) = self.journal.translation_checkpoint().await? { if checkpoint.version == self.options.version && checkpoint.input <= self.committed { @@ -700,8 +698,8 @@ impl Producer { async fn next_input(&mut self, through: u64) -> anyhow::Result> { if let Some(record) = self.options.input.take(self.position, through) { - if let SourceEvent::Event(event) = &record.event { - self.decoder.record_live(&event.event, &record.position); + if matches!(&record.event, SourceEvent::Event(_)) { + self.decoder.record_live(&record.position); } // A future gap resumes from this logical position. The live path // does not open, seek, read, or pass payloads through a journal reader. @@ -974,15 +972,17 @@ impl Producer { pending.complete = pending.ops.is_none(); pending.prepared = false; } + // Charge at least 1/64 of the budget per record, bounding both + // event count and decoded bytes with one replay-work counter. + let work = self.checkpoint_work.saturating_add( + pending + .decoded_bytes + .max(pending.input.saturating_sub(self.position)) + .max(CHECKPOINT_BYTES / CHECKPOINT_EVENTS), + ); if publish { self.phase = Boundary::Journal; - let checkpoint = if self.checkpoint_events + 1 >= CHECKPOINT_EVENTS - || pending.input.saturating_sub(self.checkpoint_position) >= CHECKPOINT_BYTES - || self - .checkpoint_work_bytes - .saturating_add(pending.decoded_bytes) - >= CHECKPOINT_BYTES - { + let checkpoint = if work >= CHECKPOINT_BYTES { self.checkpoint_context(pending.input).await? } else { None @@ -993,13 +993,9 @@ impl Producer { .await?; if checkpointed { self.checkpoint_position = pending.input; - self.checkpoint_events = 0; - self.checkpoint_work_bytes = 0; + self.checkpoint_work = 0; } else { - self.checkpoint_events += 1; - self.checkpoint_work_bytes = self - .checkpoint_work_bytes - .saturating_add(pending.decoded_bytes); + self.checkpoint_work = work; } self.committed = pending.input; self.progress @@ -1009,10 +1005,7 @@ impl Producer { // Restoring a checkpoint replays the committed suffix too. // Count that work so repeated short-lived processes cannot // keep extending the distance from the same old checkpoint. - self.checkpoint_events += 1; - self.checkpoint_work_bytes = self - .checkpoint_work_bytes - .saturating_add(pending.decoded_bytes); + self.checkpoint_work = work; } self.position = pending.input; Ok(()) @@ -1078,7 +1071,7 @@ impl Producer { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::{pipeline::delivery::InputPort, translate::SpanRow}; mod streams { @@ -1089,662 +1082,364 @@ mod tests { )); } + pub(crate) struct SourceFixture { + _dir: tempfile::TempDir, + pub(crate) options: TranslationOptions, + pub(crate) source: Arc, + } + impl SourceFixture { + pub(crate) async fn new(agent: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + let path = crate::journal::source_journal_path(dir.path(), agent, "checkpoint"); + let options = resume_options( + dir.path(), + agent, + Arc::new(tokio::sync::Mutex::new( + JournalWriter::open_path(&path).await.unwrap(), + )), + ); + let source = Translation::spawn(options.clone()).await.unwrap(); + Self { + _dir: dir, + options, + source, + } + } + pub(crate) async fn write(&self, event: &crate::wire::Envelope, live: bool) -> u64 { + let mut writer = self.options.events.lock().await; + let position = writer.append_ordered(event, None).await.unwrap(); + let through = position.through; + if live { + self.options.input.push_event(event.clone(), position); + } + through + } + pub(crate) async fn process(&self, through: u64) { + self.source.advance(through).await.unwrap(); + self.source.barrier().await.unwrap(); + assert_eq!( + self.source.journal.translated_through().await.unwrap(), + through + ); + } + async fn reopen(&mut self) { + self.source.shutdown().await; + *self.options.events.lock().await = JournalWriter::open_path(&self.options.event_path) + .await + .unwrap(); + self.options.input = SourceInput::default(); + self.source = Translation::spawn(self.options.clone()).await.unwrap(); + } + async fn checkpoint(&self) -> TranslationCheckpoint { + self.source + .journal + .translation_checkpoint() + .await + .unwrap() + .unwrap() + } + fn db(&self) -> rusqlite::Connection { + rusqlite::Connection::open(self.options.data_dir.join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&self.options.source, &self.options.session_id) + ))) + .unwrap() + } + pub(crate) async fn poison(&self, range: std::ops::Range) { + let mut bytes = tokio::fs::read(&self.options.event_path).await.unwrap(); + for byte in &mut bytes[range.start as usize..range.end as usize] { + if *byte != b'\n' { + *byte = b'x'; + } + } + tokio::fs::write(&self.options.event_path, bytes) + .await + .unwrap(); + } + } + #[tokio::test] - async fn native_source_output_matches_live_overflow_and_cold_recovery() { + async fn every_agent_recovers_equivalent_spans_at_every_native_boundary() { let mut covered = Vec::new(); - for (source, mut events, _fixtures) in streams::native_streams() { - covered.push(source.clone()); + for (agent, mut events, _files) in streams::native_streams() { + covered.push(agent.clone()); events.insert(0, events[0].clone()); for event in &mut events { event.payload["codec_padding"] = serde_json::json!("é📚history".repeat(1024)); } + let mut cases = vec![("live", 0), ("overflow", 2)]; + cases.extend((1..=events.len()).map(|split| ("cold", split))); + if agent == "debug" { + cases.extend([("old", 1), ("corrupt", 1)]); + } let mut expected = None; - for mode in ["live", "overflow", "cold"] { - let (tmp, path, writer) = source_journal(&source).await; - let options = resume_options(tmp.path(), &source, writer.clone()); - let input = options.input.clone(); - let mut translation = Translation::spawn(options).await.unwrap(); - let mut positions = Vec::new(); - for (index, event) in events.iter().enumerate() { - let through = if mode == "cold" && index > 0 { - writer.lock().await.append(event).await.unwrap() - } else { - capture(&writer, &input, event).await - }; - positions.push(through); + for (mode, split) in cases { + let mut f = SourceFixture::new(&agent).await; + let mut positions = vec![0]; + for event in &events { + positions.push(f.write(event, mode == "live" || mode == "overflow").await); } - if mode == "overflow" { - // Emulate losing the first two queued records while their - // complete persisted forms remain recoverable. - assert!(input.take(0, positions[0]).is_some()); - assert!(input.take(positions[0], positions[1]).is_some()); + let tail = *positions.last().unwrap(); + if mode == "cold" && split == 1 { + let mut reader = JournalReader::open(&f.options.event_path, tail) + .await + .unwrap() + .unwrap(); + for event in &events { + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + event.payload + ); + } } - if mode == "cold" { - translation.advance(positions[0]).await.unwrap(); - translation.barrier().await.unwrap(); - translation.shutdown().await; - // A restart must seek over this prefix, not replay it. - let mut bytes = tokio::fs::read(&path).await.unwrap(); - for byte in &mut bytes[..positions[0] as usize] { - if *byte != b'\n' { - *byte = b'x'; + assert!(tokio::fs::read_to_string(&f.options.event_path) + .await + .unwrap() + .contains("payload_zstd_v1")); + match mode { + "live" => { + f.process(positions[1]).await; + f.source.control(ControlKind::Checkpoint).await.unwrap(); + assert_eq!( + f.source.journal.translated_through().await.unwrap(), + positions[1] + ); + assert_eq!(f.options.events.lock().await.position(), tail); + f.poison(0..tail).await; + } + "overflow" => { + assert!(f.options.input.take(0, positions[1]).is_some()); + assert!(f.options.input.take(positions[1], positions[2]).is_some()); + f.poison(positions[2]..tail).await; + } + _ => { + f.process(positions[split]).await; + f.source.shutdown().await; + if mode == "cold" { + f.poison(0..positions[split]).await; + } else { + f.db() + .execute_batch(if mode == "old" { + "UPDATE translation_checkpoint SET version='old'" + } else { + "UPDATE translation_checkpoint SET state=x'00'" + }) + .unwrap(); } + f.reopen().await; } - tokio::fs::write(&path, bytes).await.unwrap(); - // New writer and input: the following stored delta must - // rely on the SQLite decoder/reducer checkpoint alone. - *writer.lock().await = JournalWriter::open_path(&path).await.unwrap(); - translation = - Translation::spawn(resume_options(tmp.path(), &source, writer.clone())) - .await - .unwrap(); } - translation - .advance(*positions.last().unwrap()) - .await - .unwrap(); - translation - .barrier() - .await - .unwrap_or_else(|error| panic!("{source} {mode}: {error}")); - translation.control(ControlKind::Finalize).await.unwrap(); - let rows = streams::comparable( - translation - .journal - .read_after(0, 10000) - .await - .unwrap() - .into_iter() - .map(|(_, op)| op) - .collect(), - ); - assert!(!rows.is_empty(), "{source} produced no observations"); + f.process(tail).await; + f.source.control(ControlKind::Finalize).await.unwrap(); + assert!(f.source.journal.translated_through().await.unwrap() > tail); + // StoredOp already includes the private correlation fields; + // compare its complete encoding rather than duplicate its codec. + let db = f.db(); + let mut query = db.prepare("SELECT value FROM ops ORDER BY value").unwrap(); + let rows: Vec> = query + .query_map([], |row| row.get(0)) + .unwrap() + .map(Result::unwrap) + .collect(); + assert!(!rows.is_empty(), "{agent} {mode} produced no observations"); if let Some(expected) = &expected { - assert_eq!(&rows, expected, "{source} {mode}"); + assert_eq!(&rows, expected, "{agent} {mode} split {split}"); } else { expected = Some(rows); } - translation.shutdown().await; + f.source.shutdown().await; } } covered.sort(); assert_eq!(covered, Registry::default_agents().sources()); } - async fn source_journal( - source: &str, - ) -> ( - tempfile::TempDir, - PathBuf, - Arc>, - ) { - let dir = tempfile::tempdir().unwrap(); - let path = crate::journal::source_journal_path(dir.path(), source, "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&path).await.unwrap(), - )); - (dir, path, writer) - } - - fn resume_options( - data_dir: &std::path::Path, - source: &str, - events: Arc>, - ) -> TranslationOptions { - TranslationOptions { - source: source.into(), - session_id: "checkpoint".into(), - namespace: crate::ids::session_namespace(source, "checkpoint"), - version: "test".into(), - data_dir: data_dir.into(), - events, - event_path: crate::journal::source_journal_path(data_dir, source, "checkpoint"), - input: SourceInput::default(), - registry: Arc::new(Registry::default_agents()), - correlation: Arc::default(), - correlation_changed: Arc::default(), - } - } - - #[tokio::test] - async fn checkpoints_cannot_absorb_captured_events_before_dispatch() { - let (tmp, _, writer) = source_journal("debug").await; - let options = resume_options(tmp.path(), "debug", writer.clone()); - let input = options.input.clone(); - let source = Translation::spawn(options).await.unwrap(); - let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{"index":0}})).unwrap(); - let first = capture(&writer, &input, &event).await; - source.advance(first).await.unwrap(); - source.barrier().await.unwrap(); - event.payload["index"] = 1.into(); - let second = capture(&writer, &input, &event).await; - // Capture has acknowledged a later event whose routing/ancestry has - // not been dispatched. A flush must not translate it ahead of that. - source.control(ControlKind::Checkpoint).await.unwrap(); - assert_eq!(source.journal.translated_through().await.unwrap(), first); - assert_eq!( - writer.lock().await.position(), - second, - "no control marker should leap over undispatched work" - ); - source.advance(second).await.unwrap(); - source.barrier().await.unwrap(); - source.control(ControlKind::Checkpoint).await.unwrap(); - assert!(source.journal.translated_through().await.unwrap() > second); - source.shutdown().await; - } - - #[tokio::test] - async fn live_translation_and_controls_do_not_read_events_back_from_disk() { - let (tmp, path, writer) = source_journal("debug").await; - let options = resume_options(tmp.path(), "debug", writer.clone()); - let input = options.input.clone(); - let translation = Translation::spawn(options).await.unwrap(); - let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ - "source":"debug", "session_id":"checkpoint", "event":"event", "ts_ms":1, - "payload":{"history":"shared history".repeat(1024),"index":0} - })) - .unwrap(); - capture(&writer, &input, &event).await; - event.payload["index"] = 1.into(); - let through = capture(&writer, &input, &event).await; - let original = tokio::fs::read(&path).await.unwrap(); - assert!(String::from_utf8_lossy(&original).contains("payload_delta_v1")); - // A journal read would fail. The complete captured records are still - // available, so normal translation and flush controls must succeed. - let mut poisoned = original.clone(); - for byte in &mut poisoned { - if *byte != b'\n' { - *byte = b'x'; - } - } - tokio::fs::write(&path, poisoned).await.unwrap(); - translation.advance(through).await.unwrap(); - translation.barrier().await.unwrap(); - translation.control(ControlKind::Checkpoint).await.unwrap(); - let rows = translation.journal.read_after(0, 32).await.unwrap(); - let inputs = rows - .iter() - .filter_map(|(_, op)| { - let (SpanOp::Insert(row) | SpanOp::Merge(row)) = op; - row.input.as_ref() - }) - .collect::>(); - assert_eq!(inputs.len(), 2); - assert_eq!(inputs[1], &event.payload); - translation.shutdown().await; - - // A cold recovery must restore the decoder bases learned from live - // records, then expand a delta written after the saved checkpoint. - event.payload["index"] = 2.into(); - let tail = writer.lock().await.append(&event).await.unwrap(); - *writer.lock().await = JournalWriter::open_path(&path).await.unwrap(); - let resumed = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) - .await - .unwrap(); - resumed.advance(tail).await.unwrap(); - resumed.barrier().await.unwrap(); - assert_eq!(resumed.journal.translated_through().await.unwrap(), tail); - let rows = resumed.journal.read_after(0, 32).await.unwrap(); - assert!(rows.iter().any(|(_, op)| { - let (SpanOp::Insert(row) | SpanOp::Merge(row)) = op; - row.input.as_ref() == Some(&event.payload) - })); - resumed.shutdown().await; - } - - async fn capture( - writer: &Arc>, - input: &SourceInput, - event: &crate::wire::Envelope, - ) -> u64 { - let mut writer = writer.lock().await; - let position = writer.append_ordered(event, None).await.unwrap(); - let through = position.through; - input.push_event(event.clone(), position); - through - } - - #[tokio::test] - async fn evicted_input_recovers_only_the_gap_then_rejoins_direct_capture() { - let (tmp, path, writer) = source_journal("debug").await; - let options = resume_options(tmp.path(), "debug", writer.clone()); - let input = options.input.clone(); - let translation = Translation::spawn(options).await.unwrap(); - let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ - "source":"debug", "session_id":"checkpoint", "event":"event", "ts_ms":1, - "payload":{"history":"shared".repeat(1024),"index":0} - })) - .unwrap(); - for batch in 0..2 { - let mut recovery_end = 0; - let mut through = 0; - for index in 0..80 { - event.payload["index"] = (batch * 80 + index).into(); - through = capture(&writer, &input, &event).await; - if index == 15 { - recovery_end = through; - } - } - // Only the first 16 records were evicted. Poison the retained - // live suffix so any attempt to read it through storage fails. - let mut stored = tokio::fs::read(&path).await.unwrap(); - for byte in &mut stored[recovery_end as usize..through as usize] { - if *byte != b'\n' { - *byte = b'x'; - } - } - tokio::fs::write(&path, stored).await.unwrap(); - translation.advance(through).await.unwrap(); - translation.barrier().await.unwrap(); - assert!(input.is_empty()); - } - let rows = translation.journal.read_after(0, 256).await.unwrap(); - let inputs = rows - .iter() - .filter_map(|(_, op)| { - let (SpanOp::Insert(row) | SpanOp::Merge(row)) = op; - row.input.as_ref() - }) - .collect::>(); - assert_eq!(inputs.len(), 160); - for (index, payload) in inputs.into_iter().enumerate() { - assert_eq!(payload["index"], index); - assert_eq!(payload["history"], event.payload["history"]); - } - translation.shutdown().await; - } - - struct BlockedCheckpoint { + struct BlockedTranslator { + checkpoint: bool, entered: Arc, release: std::sync::mpsc::Receiver<()>, } - - struct BlockedReplay { - entered: Arc, - finished: Arc, - release: std::sync::mpsc::Receiver<()>, - } - impl AgentTranslator for BlockedReplay { - fn handle( - &mut self, - _: &crate::wire::Envelope, - _: &SessionCtx, - ) -> anyhow::Result> { + impl BlockedTranslator { + fn block(&self) { self.entered.notify_one(); - let _ = self.release.recv_timeout(std::time::Duration::from_secs(5)); - self.finished.store(true, Ordering::Release); - Ok(Vec::new()) + self.release + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); } } - - #[tokio::test] - async fn synchronous_replay_work_cannot_block_the_hook_runtime() { - let (tmp, _, writer) = source_journal("debug").await; - let mut event: crate::wire::Envelope = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{}})).unwrap(); - let through = writer.lock().await.append(&event).await.unwrap(); - let options = resume_options(tmp.path(), "debug", writer.clone()); - let input = options.input.clone(); - let entered = Arc::new(Notify::new()); - let finished = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let (release, receiver) = std::sync::mpsc::channel(); - let mut producer = Producer::open(options).await.unwrap(); - producer.translator = Some(Box::new(BlockedReplay { - entered: entered.clone(), - finished: finished.clone(), - release: receiver, - })); - let started = std::time::Instant::now(); - let processing = tokio::spawn(async move { - let result = producer.advance(through).await; - (producer, result) - }); - entered.notified().await; - let elapsed = started.elapsed(); - event.ts_ms = 2; - let captured = tokio::time::timeout( - std::time::Duration::from_secs(1), - capture(&writer, &input, &event), - ) - .await; - let replay_still_blocked = !finished.load(Ordering::Acquire); - let _ = release.send(()); - let (producer, result) = processing.await.unwrap(); - assert!( - elapsed < std::time::Duration::from_secs(1), - "replay monopolized the hook runtime for {elapsed:?}" - ); - assert!(captured.unwrap() > through); - assert!( - replay_still_blocked, - "capture only completed after synchronous replay returned" - ); - result.unwrap(); - assert_eq!( - producer.journal.translated_through().await.unwrap(), - through, - "replay must retain its original stopping point" - ); - } - impl AgentTranslator for BlockedCheckpoint { + impl AgentTranslator for BlockedTranslator { fn handle( &mut self, _: &crate::wire::Envelope, _: &SessionCtx, ) -> anyhow::Result> { + if !self.checkpoint { + self.block(); + } Ok(Vec::new()) } fn save_state(&self) -> anyhow::Result>> { - self.entered.notify_one(); - self.release - .recv_timeout(std::time::Duration::from_secs(10))?; + if self.checkpoint { + self.block(); + } Ok(Some(b"{}".to_vec())) } } - #[tokio::test] - async fn slow_checkpoint_encoding_does_not_block_journal_capture_on_the_runtime() { - let (tmp, _, writer) = source_journal("debug").await; - let entered = Arc::new(Notify::new()); - let (release, resume) = std::sync::mpsc::channel(); - let mut producer = Producer::open(resume_options(tmp.path(), "debug", writer.clone())) - .await - .unwrap(); - producer.translator = Some(Box::new(BlockedCheckpoint { - entered: entered.clone(), - release: resume, - })); - let checkpoint = tokio::spawn(async move { producer.checkpoint_context(0).await }); - entered.notified().await; - let env = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"during-checkpoint","ts_ms":1,"payload":{}})).unwrap(); - let through = tokio::time::timeout(std::time::Duration::from_secs(2), async { - writer.lock().await.append(&env).await - }) - .await - .unwrap() - .unwrap(); - assert!(through > 0); - release.send(()).unwrap(); - assert!(checkpoint.await.unwrap().unwrap().is_some()); - } - - #[tokio::test] - async fn checkpoint_budget_counts_decoded_payloads_after_compression() { - let (tmp, _, writer) = source_journal("debug").await; - let source = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) - .await - .unwrap(); - let env = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{"history":"x".repeat(256 * 1024)}})).unwrap(); - let mut first = 0; - let mut tail = 0; - for index in 0..10 { - tail = writer.lock().await.append(&env).await.unwrap(); - if index == 0 { - first = tail; - } + async fn synchronous_replay_and_checkpoint_work_do_not_block_capture() { + for checkpoint in [false, true] { + let f = SourceFixture::new("debug").await; + let event = streams::native_streams().pop().unwrap().1.remove(0); + let through = f.write(&event, false).await; + let mut producer = Producer::open(f.options.clone()).await.unwrap(); + let entered = Arc::new(Notify::new()); + let (release, receiver) = std::sync::mpsc::channel(); + producer.translator = Some(Box::new(BlockedTranslator { + checkpoint, + entered: entered.clone(), + release: receiver, + })); + let started = std::time::Instant::now(); + let work = tokio::spawn(async move { + let result = if checkpoint { + producer.checkpoint_context(0).await.map(|_| ()) + } else { + producer.advance(through).await + }; + (producer, result) + }); + entered.notified().await; + assert!(started.elapsed() < std::time::Duration::from_secs(1)); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(1), f.write(&event, true)) + .await + .unwrap() + > through + ); + assert!( + !work.is_finished(), + "capture must complete before replay is released" + ); + release.send(()).unwrap(); + let (producer, result) = work.await.unwrap(); + result.unwrap(); + assert_eq!( + producer.journal.translated_through().await.unwrap(), + if checkpoint { 0 } else { through } + ); + f.source.shutdown().await; } - assert!( - tail < CHECKPOINT_BYTES, - "fixture must compress below the physical byte threshold" - ); - source.advance(tail).await.unwrap(); - source.barrier().await.unwrap(); - let checkpoint = source - .journal - .translation_checkpoint() - .await - .unwrap() - .expect("decoded work must trigger a checkpoint before 64 inputs"); - assert!(checkpoint.input > first); - source.shutdown().await; } #[tokio::test] - async fn checkpoint_window_accounts_for_committed_suffix_on_restart() { - let (tmp, _, writer) = source_journal("debug").await; - let source = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) - .await - .unwrap(); - let env = serde_json::from_value(serde_json::json!({"source":"debug","session_id":"checkpoint","event":"event","ts_ms":1,"payload":{}})).unwrap(); + async fn checkpoint_budget_counts_reconstructed_inputs_and_decoded_bytes() { + let mut f = SourceFixture::new("debug").await; + let mut event = streams::native_streams().pop().unwrap().1.remove(0); let mut tail = 0; for _ in 0..2 * CHECKPOINT_EVENTS - 1 { - tail = writer.lock().await.append(&env).await.unwrap(); + tail = f.write(&event, false).await; } - source.advance(tail).await.unwrap(); - source.barrier().await.unwrap(); - let before = source - .journal - .translation_checkpoint() - .await - .unwrap() - .unwrap() - .input; - assert!(before < tail); - let task = source.task.lock().await.take().unwrap(); + f.process(tail).await; + assert!(f.checkpoint().await.input < tail); + let task = f.source.task.lock().await.take().unwrap(); task.abort(); let _ = task.await; - tail = writer.lock().await.append(&env).await.unwrap(); - let resumed = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) - .await - .unwrap(); - resumed.advance(tail).await.unwrap(); - resumed.barrier().await.unwrap(); + tail = f.write(&event, false).await; + f.reopen().await; + f.process(tail).await; assert_eq!( - resumed - .journal - .translation_checkpoint() - .await - .unwrap() - .unwrap() - .input, + f.checkpoint().await.input, tail, - "the 63 reconstructed inputs must count toward the next periodic checkpoint" + "reconstructed suffix counts toward the next checkpoint" ); - resumed.shutdown().await; - } - - #[tokio::test] - async fn checkpoint_seeks_past_history_for_native_and_legacy_journals() { - for format in ["debug", "pi", "legacy-pi"] { - let source = if format == "legacy-pi" { "pi" } else { format }; - let (tmp, path, writer) = source_journal(source).await; - let translation = - Translation::spawn(resume_options(tmp.path(), source, writer.clone())) - .await - .unwrap(); - let mut env = crate::wire::Envelope { - source: source.into(), - session_id: "checkpoint".into(), - event: "context".into(), - ts_ms: 0, - payload: serde_json::json!({}), - route: Some(SessionRoute::default()), - config: None, - managed_run_id: None, - capture: None, - source_version: None, - plugin_version: None, - }; - let mut boundary = 0; - let mut legacy = String::new(); - for i in 0..CHECKPOINT_EVENTS + 3 { - env.ts_ms = i as i64; - env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared".repeat(1024)},{"role":"assistant","content":i}]}}); - if format == "legacy-pi" { - let mut event = env.redacted(); - event.payload["event"] - .as_object_mut() - .unwrap() - .remove("messages"); - let suffix = if i == 0 { - env.payload["event"]["messages"].clone() - } else { - serde_json::json!([{"role":"assistant","content":i}]) - }; - event.payload["event"]["_bt_messages_delta"] = serde_json::json!({"common_prefix": if i == 0 { 0 } else { 1 }, "suffix":suffix}); - legacy.push_str(&serde_json::to_string(&event).unwrap()); - legacy.push('\n'); - boundary = legacy.len() as u64; - } else { - boundary = writer.lock().await.append(&env).await.unwrap(); - } - } - if format == "legacy-pi" { - tokio::fs::write(&path, legacy).await.unwrap(); - *writer.lock().await = JournalWriter::open_path(&path).await.unwrap(); - } - translation.advance(boundary).await.unwrap(); - translation.barrier().await.unwrap(); - let checkpoint = translation - .journal - .translation_checkpoint() - .await - .unwrap() - .unwrap(); - assert!( - checkpoint.input < boundary, - "test must replay a suffix after a periodic checkpoint" - ); - let saved = RecoveryContext::decode(&checkpoint, "test", boundary).unwrap(); - assert_eq!( - saved.decoder.legacy_pi.is_empty(), - format != "legacy-pi", - "only legacy delta records should need a saved journal decoder base" - ); - let task = translation.task.lock().await.take().unwrap(); - task.abort(); - let _ = task.await; // crash, without the idle checkpoint - // Keep boundaries intact but make all pre-checkpoint JSON invalid. - // Recovery must seek over this prefix rather than decode/retranslate it. - let mut contents = tokio::fs::read(&path).await.unwrap(); - for byte in &mut contents[..checkpoint.input as usize] { - if *byte != b'\n' { - *byte = b'x'; - } - } - tokio::fs::write(&path, contents).await.unwrap(); - env.ts_ms += 1; - env.payload["event"]["messages"][1]["content"] = serde_json::json!("after restart"); - let tail = writer.lock().await.append(&env).await.unwrap(); - let resumed = Translation::spawn(resume_options(tmp.path(), source, writer.clone())) - .await - .unwrap(); - resumed.advance(tail).await.unwrap(); - resumed.barrier().await.unwrap(); - assert!( - resumed.error.lock().is_none(), - "{source}: {:?}", - resumed.error.lock() - ); - assert_eq!(resumed.journal.translated_through().await.unwrap(), tail); - resumed.shutdown().await; + let before = tail; + event.payload["history"] = "x".repeat(256 * 1024).into(); + for _ in 0..5 { + tail = f.write(&event, false).await; } + assert!( + tail - before < CHECKPOINT_BYTES, + "stored deltas must be below the physical byte threshold" + ); + f.process(tail).await; + assert!( + f.checkpoint().await.input > before, + "decoded work must trigger a checkpoint before 64 inputs" + ); + f.source.shutdown().await; } #[tokio::test] - async fn invalid_or_old_context_rebuilds_safely_from_the_source_journal() { - for alteration in [ - "UPDATE translation_checkpoint SET state=x'00'", - "UPDATE translation_checkpoint SET version='old'", - ] { - let (tmp, _, writer) = source_journal("debug").await; - let env: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ - "source":"debug", "session_id":"checkpoint", "event":"event", "ts_ms":1,"payload":{}, - })).unwrap(); - let translation = - Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) - .await - .unwrap(); - let mut tail = 0; - for _ in 0..CHECKPOINT_EVENTS { - tail = writer.lock().await.append(&env).await.unwrap(); - } - translation.advance(tail).await.unwrap(); - translation.barrier().await.unwrap(); - translation.shutdown().await; - let storage = tmp.path().join("pipeline").join(format!( - "{}.sqlite", - crate::ids::session_storage_id("debug", "checkpoint") - )); - let db = rusqlite::Connection::open(storage).unwrap(); - db.execute_batch(alteration).unwrap(); - tail = writer.lock().await.append(&env).await.unwrap(); - let resumed = Translation::spawn(resume_options(tmp.path(), "debug", writer.clone())) - .await - .unwrap(); - resumed.advance(tail).await.unwrap(); - resumed.barrier().await.unwrap(); - assert!(resumed.error.lock().is_none()); - assert_eq!(resumed.journal.translated_through().await.unwrap(), tail); - let batches: u64 = db - .query_row( - "SELECT count(*) FROM batches WHERE committed=1", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(batches, CHECKPOINT_EVENTS + 1); - resumed.shutdown().await; + async fn legacy_pi_decoder_context_resumes_without_reading_its_prefix() { + let mut f = SourceFixture::new("pi").await; + let mut event = streams::native_streams().pop().unwrap().1.remove(0); + event.source = "pi".into(); + event.event = "context".into(); + let mut contents = String::new(); + let mut positions = Vec::new(); + for i in 0..CHECKPOINT_EVENTS + 3 { + event.payload = serde_json::json!({"event":{"_bt_messages_delta":{"common_prefix":if i == 0 {0} else {1}, "suffix":[{"role":"user","content":"shared".repeat(1024)}]}}}); + contents.push_str(&serde_json::to_string(&event.redacted()).unwrap()); + contents.push('\n'); + positions.push(contents.len() as u64); } - } - - #[tokio::test] - async fn translation_checkpoint_commit_failure_does_not_move_the_source_head() { - let tmp = tempfile::tempdir().unwrap(); - let path = tmp.path().join("atomic.sqlite"); - let journal = SpanJournal::open(&path).await.unwrap(); - journal.begin(10, true).await.unwrap(); - let db = rusqlite::Connection::open(&path).unwrap(); - db.execute_batch("CREATE TRIGGER fail_context BEFORE INSERT ON translation_checkpoint BEGIN SELECT RAISE(FAIL,'checkpoint write failed'); END;").unwrap(); - let context = TranslationCheckpoint { - input: 10, - version: "test".into(), - state: vec![1, 2], - }; - assert!(journal - .commit_with_context(10, &Default::default(), Some(context.clone())) - .await - .is_err()); - assert_eq!(journal.translated_through().await.unwrap(), 0); - assert!(journal.translation_checkpoint().await.unwrap().is_none()); - db.execute_batch("DROP TRIGGER fail_context;").unwrap(); - journal - .commit_with_context(10, &Default::default(), Some(context)) + tokio::fs::write(&f.options.event_path, contents) .await .unwrap(); - assert_eq!(journal.translated_through().await.unwrap(), 10); - assert_eq!( - journal - .translation_checkpoint() + f.reopen().await; + let tail = *positions.last().unwrap(); + f.process(tail).await; + let saved = f.checkpoint().await; + assert!(saved.input < tail); + assert!(!RecoveryContext::decode(&saved, "test", tail) + .unwrap() + .decoder + .legacy_pi + .is_empty()); + let task = f.source.task.lock().await.take().unwrap(); + task.abort(); + let _ = task.await; + f.poison(0..saved.input).await; + f.reopen().await; + f.process(tail).await; + event.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"native"}],"_bt_messages_delta":{"native_field":"preserve"}}}); + let native = f.write(&event, false).await; + f.process(native).await; + let mut reader = + JournalReader::resume(&f.options.event_path, tail, native, Default::default()) .await - .unwrap() - .unwrap() - .input, - 10 + .unwrap(); + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + event.payload ); + f.source.shutdown().await; } - #[tokio::test] - async fn shared_storage_stays_open_until_the_last_route_finishes_draining() { - let tmp = tempfile::tempdir().unwrap(); - let event_path = tmp.path().join("events.ndjson"); - let translation = Translation::spawn(TranslationOptions { - source: "debug".into(), - session_id: "draining".into(), - namespace: "draining".into(), + fn resume_options( + data_dir: &std::path::Path, + source: &str, + events: Arc>, + ) -> TranslationOptions { + TranslationOptions { + source: source.into(), + session_id: "checkpoint".into(), + namespace: crate::ids::session_namespace(source, "checkpoint"), version: "test".into(), - data_dir: tmp.path().into(), - events: Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&event_path).await.unwrap(), - )), - event_path, + data_dir: data_dir.into(), + events, + event_path: crate::journal::source_journal_path(data_dir, source, "checkpoint"), input: SourceInput::default(), registry: Arc::new(Registry::default_agents()), - correlation: Arc::new(crate::correlation::CorrelationRegistry::default()), - correlation_changed: Arc::new(Notify::new()), - }) - .await - .unwrap(); + correlation: Arc::default(), + correlation_changed: Arc::default(), + } + } + + #[tokio::test] + async fn shared_storage_stays_open_until_the_last_route_finishes_draining() { + let f = SourceFixture::new("debug").await; + let translation = &f.source; translation.routes.lock().active = 2; assert!(!translation.begin_route_shutdown()); assert!(translation.begin_route_shutdown()); @@ -1756,26 +1451,9 @@ mod tests { #[tokio::test] async fn recreated_observer_restores_a_staged_session_root() { - let tmp = tempfile::tempdir().unwrap(); - let event_path = tmp.path().join("events.ndjson"); - let correlation = Arc::new(crate::correlation::CorrelationRegistry::default()); - let translation = Translation::spawn(TranslationOptions { - source: "debug".into(), - session_id: "staged".into(), - namespace: "staged".into(), - version: "test".into(), - data_dir: tmp.path().into(), - events: Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&event_path).await.unwrap(), - )), - event_path, - input: SourceInput::default(), - registry: Arc::new(Registry::default_agents()), - correlation: correlation.clone(), - correlation_changed: Arc::new(Notify::new()), - }) - .await - .unwrap(); + let f = SourceFixture::new("debug").await; + let translation = &f.source; + let correlation = &f.options.correlation; let mut roots = Roots::default(); roots .process(&vec![ @@ -1904,12 +1582,11 @@ mod tests { } #[tokio::test] async fn failed_append_and_commit_retain_the_built_chunk() { - let (tmp, _, writer) = source_journal("debug").await; - let mut producer = Producer::open(resume_options(tmp.path(), "debug", writer)) - .await - .unwrap(); + let f = SourceFixture::new("debug").await; + f.source.shutdown().await; + let mut producer = Producer::open(f.options.clone()).await.unwrap(); let journal = producer.journal.clone(); - let path = tmp.path().join("pipeline").join(format!( + let path = f.options.data_dir.as_path().join("pipeline").join(format!( "{}.sqlite", crate::ids::session_storage_id("debug", "checkpoint") )); diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 33518ac3..fece76df 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -6,6 +6,7 @@ use super::*; use std::collections::VecDeque; /// Captures remain ordered and bounded. Evicted entries are /// already durable; their source journal's high-water mark is replayed first. +#[derive(Default)] pub(super) struct CaptureBuffer { pub(super) events: VecDeque, pub(super) evicted: HashMap, @@ -18,13 +19,6 @@ pub(super) struct EvictedRange { } impl CaptureBuffer { - pub(super) fn new() -> Self { - Self { - events: VecDeque::new(), - evicted: HashMap::new(), - } - } - fn push(&mut self, data_dir: &std::path::Path, event: PendingEvent) { if self.events.len() == INGRESS_QUEUE_CAPACITY { if let Some(oldest) = self.events.pop_front() { @@ -88,7 +82,7 @@ pub(super) async fn replay_evicted( let mut reader = match cursors.remove(path) { Some(reader) => reader, None => match resume_discovery(daemon, path, range.from).await { - Ok(Some((reader, _, _))) => reader, + Ok(Some((reader, _))) => reader, _ => journal::JournalReader::open(path, range.from) .await? .ok_or_else(|| anyhow::anyhow!("evicted capture journal disappeared"))?, diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 6c7d5dba..23a2b222 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -438,28 +438,7 @@ mod tests { )); } - struct TestAuth; - - #[async_trait] - impl AuthProvider for TestAuth { - async fn resolve( - &self, - selection: &AuthSelection, - _: AuthResolveReason, - ) -> anyhow::Result { - Ok(AuthLease { - selection: selection.clone().canonicalized()?, - auth: BackendAuth { - token: "test".into(), - api_url: Some("http://127.0.0.1.invalid".into()), - app_url: None, - org_name: Some("test".into()), - org_id: Some("test".into()), - }, - expires_at_ms: None, - }) - } - } + use super::super::recovery::tests::TestAuth; fn event(session: &str, route: SessionRoute) -> Envelope { Envelope { @@ -477,93 +456,43 @@ mod tests { } } - #[tokio::test] - async fn capture_is_durable_while_startup_dispatch_is_blocked() { - let tmp = tempfile::tempdir().unwrap(); + fn paused_ingress(root: &std::path::Path) -> (Arc, oneshot::Sender<()>) { let (ready, startup) = oneshot::channel(); - let daemon = Daemon::new_with_startup( - debug_serve_options("test", tmp.path()), - tmp.path().into(), - Some(startup), - ); - let env = event("during-startup", SessionRoute::default()); - assert!( - tokio::time::timeout(Duration::from_secs(1), daemon.capture_event(env.clone())) - .await - .unwrap() - .unwrap() - ); - let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); - assert!(tokio::fs::read_to_string(path) - .await - .unwrap() - .contains("during-startup")); - assert!(daemon.translations.lock().is_empty()); - drain_capture_buffer(&daemon).await.unwrap(); - ready.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(3), daemon.ingress_barrier()) - .await - .unwrap(); - assert!(!daemon.translations.lock().is_empty()); - drain_all(&daemon).await; + ( + Daemon::new_with_startup( + debug_serve_options("test", root), + root.into(), + Some(startup), + ), + ready, + ) } #[tokio::test] - async fn shutdown_observes_transcript_registered_by_queued_capture() { + async fn capture_is_durable_during_startup_and_recovery_stops_before_it() { let tmp = tempfile::tempdir().unwrap(); - let transcript = tmp.path().join("transcript.jsonl"); - tokio::fs::write(&transcript, b"{}\n").await.unwrap(); - let (ready, startup) = oneshot::channel(); - let daemon = Daemon::new_with_startup( - debug_serve_options("test", tmp.path()), - tmp.path().into(), - Some(startup), - ); - let mut env = event("late-transcript", SessionRoute::default()); - env.source = "claude-code".into(); - env.event = "Stop".into(); - env.payload = serde_json::json!({"transcript_path": transcript}); - daemon.capture_event(env.clone()).await.unwrap(); - tokio::fs::write(&transcript, b"{}\n{}\n").await.unwrap(); - - let shutdown_daemon = daemon.clone(); - let shutdown = tokio::spawn(async move { drain_all(&shutdown_daemon).await }); - while !daemon.quiescing.load(Ordering::SeqCst) { - tokio::task::yield_now().await; - } - tokio::task::yield_now().await; - drain_capture_buffer(&daemon).await.unwrap(); - ready.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(10), shutdown) + let mut env = event("resuming", SessionRoute::default()); + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + let old_end = JournalWriter::open_path(&path) .await .unwrap() + .append(&env) + .await .unwrap(); - let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); - let content = tokio::fs::read_to_string(path).await.unwrap(); + let (daemon, ready) = paused_ingress(tmp.path()); + env.ts_ms = 2; + env.event = "next".into(); assert!( - content.contains("TranscriptUpdate"), - "late transcript must be journaled before shutdown" - ); - } - - #[tokio::test] - async fn startup_recovery_dispatches_old_session_before_new_capture() { - let tmp = tempfile::tempdir().unwrap(); - let env = event("resuming", SessionRoute::default()); - let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let old_end = writer.append(&env).await.unwrap(); - drop(writer); - let (ready, startup) = oneshot::channel(); - let daemon = Daemon::new_with_startup( - debug_serve_options("test", tmp.path()), - tmp.path().into(), - Some(startup), + tokio::time::timeout(Duration::from_secs(1), daemon.capture_event(env)) + .await + .unwrap() + .unwrap() ); - let mut next = env.clone(); - next.event = "next".into(); - next.ts_ms = 2; - assert!(daemon.capture_event(next).await.unwrap()); + assert!(tokio::fs::read_to_string(&path) + .await + .unwrap() + .contains("next")); + assert!(daemon.translations.lock().is_empty()); let recovered = discover_unprocessed_journals(&daemon).await; assert_eq!(recovered.len(), 1); assert_eq!(recovered[0].journal_through, old_end); @@ -572,181 +501,151 @@ mod tests { } drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(3), daemon.ingress_barrier()) - .await - .unwrap(); + daemon.settle_ingress().await; let source = daemon.translations.lock().values().next().unwrap().clone(); - source.barrier().await.unwrap(); + assert!(source.barrier().await.unwrap() > 0); let storage = tmp.path().join("pipeline").join(format!( "{}.sqlite", - crate::ids::session_storage_id(&env.source, &env.session_id) + crate::ids::session_storage_id("debug", "resuming") )); let partition = crate::pipeline::journal::SpanJournal::open(&storage) .await .unwrap(); - let through = partition.translated_through().await.unwrap(); - assert!(through > old_end); - assert_eq!(through, journal::JournalReader::recorded_len(&path).await); - drain_all(&daemon).await; - } - - #[tokio::test] - async fn startup_overflow_replays_oldest_events_before_the_buffered_tail() { - let tmp = tempfile::tempdir().unwrap(); - let (ready, startup) = oneshot::channel(); - let daemon = Daemon::new_with_startup( - debug_serve_options("test", tmp.path()), - tmp.path().into(), - Some(startup), - ); - let env = event("overflow", SessionRoute::default()); - let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); - for i in 0..(INGRESS_QUEUE_CAPACITY + 6) { - let mut next = env.clone(); - next.ts_ms = i as i64; - next.payload = serde_json::json!({"index": i}); - assert!(daemon.capture_event(next).await.unwrap()); - } - { - let state = daemon.capture_buffer.lock(); - let buffer = &state; - assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); - assert_eq!(buffer.events.front().unwrap().env.payload["index"], 6); - } - drain_capture_buffer(&daemon).await.unwrap(); - ready.send(()).unwrap(); - daemon.ingress_barrier().await; - let source = daemon.translations.lock().values().next().unwrap().clone(); - source.barrier().await.unwrap(); - let storage = tmp.path().join("pipeline").join(format!( - "{}.sqlite", - crate::ids::session_storage_id(&env.source, &env.session_id) - )); - let db = rusqlite::Connection::open(storage).unwrap(); - let count: u64 = db - .query_row( - "SELECT count(*) FROM batches WHERE committed=1", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(count, (INGRESS_QUEUE_CAPACITY + 6) as u64); - let partition = crate::pipeline::journal::SpanJournal::open( - &tmp.path().join("pipeline").join(format!( - "{}.sqlite", - crate::ids::session_storage_id(&env.source, &env.session_id) - )), - ) - .await - .unwrap(); assert_eq!( partition.translated_through().await.unwrap(), journal::JournalReader::recorded_len(&path).await ); - drop(db); drain_all(&daemon).await; } #[tokio::test] - async fn startup_overflow_advances_again_while_evicted_records_are_catching_up() { + async fn repeated_overflow_preserves_each_session_and_its_compressed_context() { let tmp = tempfile::tempdir().unwrap(); - let (ready, startup) = oneshot::channel(); - let daemon = Daemon::new_with_startup( - debug_serve_options("test", tmp.path()), - tmp.path().into(), - Some(startup), - ); - let env = event("chasing", SessionRoute::default()); - for i in 0..(INGRESS_QUEUE_CAPACITY + 2) { - let mut next = env.clone(); - next.ts_ms = i as i64; - next.payload = serde_json::json!({"index": i}); - assert!(daemon.capture_event(next).await.unwrap()); + let (daemon, ready) = paused_ingress(tmp.path()); + // Four journals interleave through two eviction ranges and a live tail. + let mut envs = Vec::new(); + for source in ["debug", "pi"] { + for session in ["first", "second"] { + let mut env = event(session, SessionRoute::default()); + env.source = source.into(); + env.event = "context".into(); + envs.push(env); + } } - let CaptureWork::Evicted(first) = daemon.next_capture_work() else { - panic!("the oldest events were not evicted"); - }; - for i in INGRESS_QUEUE_CAPACITY + 2..INGRESS_QUEUE_CAPACITY + 7 { - let mut next = env.clone(); - next.ts_ms = i as i64; - next.payload = serde_json::json!({"index": i}); - assert!(daemon.capture_event(next).await.unwrap()); + let mut first = None; + let total = 2 * INGRESS_QUEUE_CAPACITY + 12; + for index in 0..total { + let mut env = envs[index % envs.len()].clone(); + env.ts_ms = index as i64; + env.payload = serde_json::json!({"index":index,"event":{"messages":[{"role":"user","content":"history".repeat(1024)},{"role":"assistant","content":index}]}}); + daemon.capture_event(env).await.unwrap(); + if index + 1 == 2 * INGRESS_QUEUE_CAPACITY { + let CaptureWork::Evicted(bounds) = daemon.next_capture_work() else { + panic!("missing first eviction"); + }; + assert_eq!(bounds.len(), envs.len()); + first = Some(bounds); + } } { - let state = daemon.capture_buffer.lock(); - let buffer = &state; + let buffer = daemon.capture_buffer.lock(); assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); - assert_eq!(buffer.events.front().unwrap().env.payload["index"], 7); + assert_eq!( + buffer.events.front().unwrap().env.payload["index"], + total - INGRESS_QUEUE_CAPACITY + ); } let mut cursors = HashMap::new(); - replay_evicted(&daemon, &mut cursors, &first).await.unwrap(); + replay_evicted(&daemon, &mut cursors, &first.unwrap()) + .await + .unwrap(); let CaptureWork::Evicted(second) = daemon.next_capture_work() else { - panic!("new evictions did not advance recovery"); + panic!("new captures must extend recovery"); }; replay_evicted(&daemon, &mut cursors, &second) .await .unwrap(); drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); - daemon.ingress_barrier().await; - let source = daemon.translations.lock().values().next().unwrap().clone(); - source.barrier().await.unwrap(); - let storage = tmp.path().join("pipeline").join(format!( - "{}.sqlite", - crate::ids::session_storage_id(&env.source, &env.session_id) - )); - let db = rusqlite::Connection::open(storage).unwrap(); - let count: u64 = db - .query_row( - "SELECT count(*) FROM batches WHERE committed=1", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(count, (INGRESS_QUEUE_CAPACITY + 7) as u64); - drop(db); + daemon.settle_ingress().await; + for (slot, env) in envs.iter().enumerate() { + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + assert!(tokio::fs::read_to_string(&path) + .await + .unwrap() + .contains("payload_zstd_v1")); + let storage = tmp.path().join("pipeline").join(format!( + "{}.sqlite", + crate::ids::session_storage_id(&env.source, &env.session_id) + )); + let partition = crate::pipeline::journal::SpanJournal::open(&storage) + .await + .unwrap(); + assert_eq!( + partition.translated_through().await.unwrap(), + journal::JournalReader::recorded_len(&path).await + ); + let db = rusqlite::Connection::open(storage).unwrap(); + let count: usize = db + .query_row( + "SELECT count(*) FROM batches WHERE committed=1", + [], + |row| row.get(0), + ) + .unwrap(); + let expected: Vec<_> = (slot..total).step_by(envs.len()).collect(); + assert_eq!(count, expected.len()); + if env.source == "debug" { + let actual: Vec<_> = partition + .read_after(0, total) + .await + .unwrap() + .into_iter() + .filter_map(|(_, op)| { + let (SpanOp::Insert(row) | SpanOp::Merge(row)) = op; + row.input?.get("index")?.as_u64() + }) + .collect(); + assert_eq!( + actual, + expected.into_iter().map(|i| i as u64).collect::>() + ); + } + } drain_all(&daemon).await; } #[tokio::test] - async fn overflow_snapshot_excludes_unacknowledged_writes() { + async fn shutdown_observes_transcript_registered_by_queued_capture() { let tmp = tempfile::tempdir().unwrap(); - let (ready, startup) = oneshot::channel(); - let mut opts = debug_serve_options("test", tmp.path()); - opts.auth_provider = Some(Arc::new(TestAuth)); - let daemon = Daemon::new_with_startup(opts, tmp.path().into(), Some(startup)); - let mut env = event("durable-bound", SessionRoute::default()); - env.payload = serde_json::json!({"index":0}); + let transcript = tmp.path().join("transcript.jsonl"); + tokio::fs::write(&transcript, b"{}\n").await.unwrap(); + let (daemon, ready) = paused_ingress(tmp.path()); + let mut env = event("late-transcript", SessionRoute::default()); + env.source = "claude-code".into(); + env.event = "Stop".into(); + env.payload = serde_json::json!({"transcript_path": transcript}); daemon.capture_event(env.clone()).await.unwrap(); - let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); - let acknowledged = tokio::fs::read(&path).await.unwrap(); - let through = acknowledged.len() as u64; - // OS-visible bytes need not have completed fsync. Model a complete - // record written after the writer's acknowledged durable position. - env.payload = serde_json::json!({"index":1}); - let mut unacknowledged = acknowledged.clone(); - unacknowledged.extend(serde_json::to_vec(&env.redacted()).unwrap()); - unacknowledged.push(b'\n'); - tokio::fs::write(&path, unacknowledged).await.unwrap(); + tokio::fs::write(&transcript, b"{}\n{}\n").await.unwrap(); + + let shutdown_daemon = daemon.clone(); + let shutdown = tokio::spawn(async move { drain_all(&shutdown_daemon).await }); + while !daemon.quiescing.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + tokio::task::yield_now().await; drain_capture_buffer(&daemon).await.unwrap(); - let source = daemon.translations.lock().values().next().unwrap().clone(); - source.barrier().await.unwrap(); - let storage = tmp.path().join("pipeline").join(format!( - "{}.sqlite", - crate::ids::session_storage_id(&env.source, &env.session_id) - )); - let partition = crate::pipeline::journal::SpanJournal::open(&storage) + ready.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), shutdown) .await + .unwrap() .unwrap(); - assert_eq!( - partition.translated_through().await.unwrap(), - through, - "recovery must not commit beyond the durable capture head" + let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); + let content = tokio::fs::read_to_string(path).await.unwrap(); + assert!( + content.contains("TranscriptUpdate"), + "late transcript must be journaled before shutdown" ); - tokio::fs::write(&path, acknowledged).await.unwrap(); - ready.send(()).unwrap(); - daemon.settle_ingress().await; - drain_all(&daemon).await; } #[tokio::test] @@ -812,106 +711,6 @@ mod tests { drain_all(&daemon).await; } - #[tokio::test] - async fn startup_overflow_tracks_each_source_journal_independently() { - let tmp = tempfile::tempdir().unwrap(); - let (ready, startup) = oneshot::channel(); - let daemon = Daemon::new_with_startup( - debug_serve_options("test", tmp.path()), - tmp.path().into(), - Some(startup), - ); - for i in 0..(INGRESS_QUEUE_CAPACITY + 6) { - let session = if i % 2 == 0 { "first" } else { "second" }; - let mut env = event(session, SessionRoute::default()); - env.ts_ms = i as i64; - env.payload = serde_json::json!({"index": i}); - assert!(daemon.capture_event(env).await.unwrap()); - } - { - let state = daemon.capture_buffer.lock(); - let buffer = &state; - assert_eq!(buffer.events.len(), INGRESS_QUEUE_CAPACITY); - } - drain_capture_buffer(&daemon).await.unwrap(); - ready.send(()).unwrap(); - daemon.ingress_barrier().await; - let sources: Vec<_> = daemon.translations.lock().values().cloned().collect(); - for source in sources { - source.barrier().await.unwrap(); - } - for session in ["first", "second"] { - let storage = tmp.path().join("pipeline").join(format!( - "{}.sqlite", - crate::ids::session_storage_id("debug", session) - )); - let db = rusqlite::Connection::open(storage).unwrap(); - let count: u64 = db - .query_row( - "SELECT count(*) FROM batches WHERE committed=1", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(count, ((INGRESS_QUEUE_CAPACITY + 6) / 2) as u64); - } - drain_all(&daemon).await; - } - - #[tokio::test] - async fn startup_overflow_keeps_pi_context_across_recovery_advances() { - let tmp = tempfile::tempdir().unwrap(); - let (ready, startup) = oneshot::channel(); - let daemon = Daemon::new_with_startup( - debug_serve_options("test", tmp.path()), - tmp.path().into(), - Some(startup), - ); - let mut env = event("pi-overflow", SessionRoute::default()); - env.source = "pi".into(); - env.event = "context".into(); - for i in 0..(INGRESS_QUEUE_CAPACITY + 2) { - env.ts_ms = i as i64; - env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); - assert!(daemon.capture_event(env.clone()).await.unwrap()); - } - let CaptureWork::Evicted(first) = daemon.next_capture_work() else { - panic!("missing first overflow") - }; - let mut cursors = HashMap::new(); - replay_evicted(&daemon, &mut cursors, &first).await.unwrap(); - for i in INGRESS_QUEUE_CAPACITY + 2..INGRESS_QUEUE_CAPACITY + 4 { - env.ts_ms = i as i64; - env.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"shared"},{"role":"assistant","content":i}]}}); - assert!(daemon.capture_event(env.clone()).await.unwrap()); - } - let CaptureWork::Evicted(second) = daemon.next_capture_work() else { - panic!("missing second overflow") - }; - replay_evicted(&daemon, &mut cursors, &second) - .await - .unwrap(); - drain_capture_buffer(&daemon).await.unwrap(); - ready.send(()).unwrap(); - daemon.ingress_barrier().await; - let source = daemon.translations.lock().values().next().unwrap().clone(); - source.barrier().await.unwrap(); - let storage = tmp.path().join("pipeline").join(format!( - "{}.sqlite", - crate::ids::session_storage_id(&env.source, &env.session_id) - )); - let db = rusqlite::Connection::open(storage).unwrap(); - let count: u64 = db - .query_row( - "SELECT count(*) FROM batches WHERE committed=1", - [], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(count, (INGRESS_QUEUE_CAPACITY + 4) as u64); - drop(db); - drain_all(&daemon).await; - } #[tokio::test] async fn retention_requires_complete_inactive_partitions_and_nonempty_heads() { for scenario in [ diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index b9a5948d..4224f12f 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -187,7 +187,7 @@ impl Daemon { correlation_changed: Arc::new(Notify::new()), ingress_tx, startup_cutoffs: Mutex::new(startup_cutoffs), - capture_buffer: Mutex::new(CaptureBuffer::new()), + capture_buffer: Mutex::new(CaptureBuffer::default()), capture_sequence: AtomicU64::new(0), started: Instant::now(), last_activity: Mutex::new(Instant::now()), diff --git a/bt-daemon/src/server/recovery.rs b/bt-daemon/src/server/recovery.rs index 2dc4e9e8..348a7561 100644 --- a/bt-daemon/src/server/recovery.rs +++ b/bt-daemon/src/server/recovery.rs @@ -8,13 +8,7 @@ pub(super) async fn resume_discovery( daemon: &Daemon, path: &std::path::Path, through: u64, -) -> anyhow::Result< - Option<( - journal::JournalReader, - u64, - Vec, - )>, -> { +) -> anyhow::Result)>> { let Some((_, id)) = path .file_stem() .and_then(|name| name.to_str()) @@ -58,11 +52,7 @@ pub(super) async fn resume_discovery( "checkpoint contains another source session" ); let reader = journal::JournalReader::resume(path, input, through, context.decoder).await?; - Ok(Some(( - reader, - input, - context.templates.into_values().collect(), - ))) + Ok(Some((reader, context.templates.into_values().collect()))) } pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec { @@ -98,8 +88,9 @@ pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec

= HashMap::new(); let mut latest_by_route: HashMap = HashMap::new(); match resume_discovery(daemon, &path, recorded_len).await { - Ok(Some((resumed, input, templates))) => { + Ok(Some((resumed, templates))) => { reader = resumed; + let input = reader.position(); before = input; for template in templates { let env = journal::envelope_from_redacted(template); @@ -307,11 +298,11 @@ pub(super) async fn recover_unprocessed_journals(daemon: &Arc) { } #[cfg(test)] -mod tests { +pub(super) mod tests { use super::*; use crate::translate::{SpanOp, SpanRow}; - struct TestAuth; + pub(crate) struct TestAuth; #[async_trait::async_trait] impl AuthProvider for TestAuth { async fn resolve( @@ -335,12 +326,8 @@ mod tests { #[tokio::test] async fn startup_discovery_uses_the_session_checkpoint_instead_of_scanning_history() { - use crate::pipeline::translation::{Translation, TranslationOptions}; - let tmp = tempfile::tempdir().unwrap(); - let raw = journal::source_journal_path(tmp.path(), "debug", "checkpoint"); - let writer = Arc::new(tokio::sync::Mutex::new( - JournalWriter::open_path(&raw).await.unwrap(), - )); + use crate::pipeline::translation::tests::SourceFixture; + let f = SourceFixture::new("debug").await; let mut env: Envelope = serde_json::from_value(serde_json::json!({ "source":"debug", "session_id":"checkpoint", "event":"old", "ts_ms":1, "payload":{}, "route":SessionRoute::default(), @@ -349,41 +336,20 @@ mod tests { let mut through = 0; for i in 0..64 { env.ts_ms = i; - through = writer.lock().await.append(&env).await.unwrap(); - } - let translation = Translation::spawn(TranslationOptions { - source: "debug".into(), - session_id: "checkpoint".into(), - namespace: crate::ids::session_namespace("debug", "checkpoint"), - version: "test".into(), - data_dir: tmp.path().into(), - events: writer.clone(), - event_path: raw.clone(), - input: crate::pipeline::input::SourceInput::default(), - registry: Arc::new(Registry::default_agents()), - correlation: Arc::default(), - correlation_changed: Arc::default(), - }) - .await - .unwrap(); - translation.advance(through).await.unwrap(); - translation.barrier().await.unwrap(); - translation.shutdown().await; - let mut contents = tokio::fs::read(&raw).await.unwrap(); - for byte in &mut contents[..through as usize] { - if *byte != b'\n' { - *byte = b'x'; - } + through = f.write(&env, false).await; } - tokio::fs::write(&raw, contents).await.unwrap(); + f.process(through).await; + f.source.shutdown().await; + f.poison(0..through).await; env.event = "after".into(); - let tail = writer.lock().await.append(&env).await.unwrap(); - let daemon = Daemon::new(debug_serve_options("test", tmp.path()), tmp.path().into()); - let (_, start, _) = resume_discovery(&daemon, &raw, tail) - .await - .unwrap() - .unwrap(); - assert_eq!(start, through); + let tail = f.write(&env, false).await; + let raw = &f.options.event_path; + let daemon = Daemon::new( + debug_serve_options("test", &f.options.data_dir), + f.options.data_dir.clone(), + ); + let (reader, _) = resume_discovery(&daemon, raw, tail).await.unwrap().unwrap(); + assert_eq!(reader.position(), through); let candidates = discover_unprocessed_journals(&daemon).await; assert_eq!(candidates.len(), 1); assert_eq!(candidates[0].env.event, "after"); diff --git a/bt-daemon/src/translate/pi.rs b/bt-daemon/src/translate/pi.rs index 6ea26e26..93e90746 100644 --- a/bt-daemon/src/translate/pi.rs +++ b/bt-daemon/src/translate/pi.rs @@ -1414,37 +1414,3 @@ fn bash_status(tool: &str, v: Option<&Value>) -> Option { .any(|prefix| status.starts_with(prefix)) .then(|| status.to_string()) } - -#[cfg(test)] -mod legacy_journal_tests { - use super::*; - #[test] - fn legacy_request_config_normalization_is_idempotent() { - let schema = serde_json::json!({"type": "object"}); - for payload in [ - serde_json::json!({ - "max_tokens": 1024, - "tools": [ - {"name": "read", "input_schema": schema, "strict": true}, - {"type": "web_search_20250305", "name": "web_search"}, - ], - "tool_choice": {"type": "tool", "name": "read", "disable_parallel_tool_use": true}, - }), - serde_json::json!({"config": { - "temperature": 0.5, - "tools": [{"functionDeclarations": [{"name": "read", "parameters": schema}]}, {"googleSearch": {}}], - "toolConfig": {"functionCallingConfig": {"mode": "ANY"}}, - }}), - serde_json::json!({ - "toolConfig": {"tools": [{"toolSpec": {"name": "read", "inputSchema": {"json": schema}}}]}, - "inferenceConfig": {"maxTokens": 512}, - }), - ] { - let journaled = serde_json::Value::Object(request_config(&payload)); - assert_eq!( - serde_json::Value::Object(request_config(&journaled)), - journaled - ); - } - } -} diff --git a/bt-daemon/src/transport.rs b/bt-daemon/src/transport.rs index 99436bb0..a1152b2c 100644 --- a/bt-daemon/src/transport.rs +++ b/bt-daemon/src/transport.rs @@ -269,21 +269,12 @@ pub(crate) fn cleanup(_endpoint: &Path) {} #[cfg(all(test, unix))] mod unix_tests { - #[test] - fn endpoint_lock_survives_an_unresponsive_owner_and_is_reusable() { - let tmp = tempfile::tempdir().unwrap(); - let endpoint = tmp.path().join("daemon.sock"); - let owner = super::lock_endpoint(&endpoint).unwrap().unwrap(); - assert!(super::lock_endpoint(&endpoint).unwrap().is_none()); - drop(owner); - assert!(super::lock_endpoint(&endpoint).unwrap().is_some()); - } - #[tokio::test] async fn startup_owner_excludes_a_contender_before_socket_bind() { let tmp = tempfile::tempdir().unwrap(); let endpoint = tmp.path().join("daemon.sock"); let owner = super::lock_endpoint(&endpoint).unwrap().unwrap(); + assert!(super::lock_endpoint(&endpoint).unwrap().is_none()); assert!(!endpoint.exists()); let contender_path = endpoint.clone(); let mut contender = tokio::spawn(async move { diff --git a/bt-daemon/tests/support/translation_streams.rs b/bt-daemon/tests/support/translation_streams.rs index a04a6119..0de17dd5 100644 --- a/bt-daemon/tests/support/translation_streams.rs +++ b/bt-daemon/tests/support/translation_streams.rs @@ -1,4 +1,4 @@ -// Native event streams shared by reducer and storage-equivalence tests. +// Native event streams for storage and recovery equivalence tests. use bt_daemon as fixture_sdk; #[allow(dead_code)] mod distributed { @@ -12,19 +12,7 @@ use bt_daemon::wire::Envelope; use distributed::{AgentKind, DistributedFixtures, ProcessTree}; use serde_json::{json, Value}; fn event(source: &str, name: &str, ts: i64, payload: Value) -> Envelope { - Envelope { - source: source.into(), - source_version: None, - plugin_version: None, - session_id: "checkpoint".into(), - event: name.into(), - ts_ms: ts, - managed_run_id: None, - capture: None, - payload, - route: None, - config: None, - } + serde_json::from_value(json!({"source":source,"session_id":"checkpoint","event":name,"ts_ms":ts,"payload":payload})).unwrap() } // Keep the fixtures alive while consuming a stream: Codex events refer to // transcript files owned by this guard. @@ -32,121 +20,72 @@ pub type NativeStream = (String, Vec, Option); pub fn native_streams() -> Vec { let mut cases = Vec::::new(); - { - for kind in AgentKind::ALL { - let mut fixtures = DistributedFixtures::new(); - let tree = ProcessTree::root(1234); - let mut events = - fixtures.start_turn(kind, "checkpoint", &tree, "first", 1_700_000_000_000); - events.push(fixtures.open_tool( - kind, - "checkpoint", - &tree, - "call", - "work", - 1_700_000_000_010, - )); - events.push(fixtures.close_tool( - kind, - "checkpoint", - &tree, - "call", - "work", - "result", - 1_700_000_000_020, - )); - events.push(fixtures.close_session( - kind, - "checkpoint", - &tree, - "complete", - 1_700_000_000_030, - )); - events.extend(fixtures.start_turn( - kind, - "checkpoint", - &tree, - "second", - 1_700_000_000_040, - )); - let source = events[0].source.clone(); - cases.push((source.clone(), events, Some(fixtures))); - } + for kind in AgentKind::ALL { + let mut fixtures = DistributedFixtures::new(); + let tree = ProcessTree::root(1234); + let id = "checkpoint"; + let t = 1_700_000_000_000; + let mut events = fixtures.start_turn(kind, id, &tree, "first", t); + events.push(fixtures.open_tool(kind, id, &tree, "call", "work", t + 10)); + events.push(fixtures.close_tool(kind, id, &tree, "call", "work", "result", t + 20)); + events.push(fixtures.close_session(kind, id, &tree, "complete", t + 30)); + events.extend(fixtures.start_turn(kind, id, &tree, "second", t + 40)); + cases.push((events[0].source.clone(), events, Some(fixtures))); } - { - let events = include_str!("../fixtures/muse-hooks-1.1.1.jsonl") + let events = include_str!("../fixtures/muse-hooks-1.1.1.jsonl") + .lines() + .enumerate() + .map(|(i, line)| { + let payload: Value = serde_json::from_str(line).unwrap(); + let name = payload["hook_event_name"].as_str().unwrap().to_owned(); + event("muse", &name, i as i64 + 1, payload) + }) + .collect(); + cases.push(("muse".to_owned(), events, None)); + let events = + include_str!("../fixtures/cursor/real-cli/interactive-multi-turn-success/hooks.ndjson") .lines() - .enumerate() - .map(|(i, line)| { - let payload: Value = serde_json::from_str(line).unwrap(); - let name = payload["hook_event_name"].as_str().unwrap().to_owned(); - event("muse", &name, i as i64 + 1, payload) + .map(|line| { + let capture: Value = serde_json::from_str(line).unwrap(); + let payload = capture["payload"].clone(); + event( + "cursor", + payload["hook_event_name"].as_str().unwrap(), + capture["received_at_ms"].as_i64().unwrap(), + payload.clone(), + ) }) .collect(); - cases.push(("muse".to_owned(), events, None)); + cases.push(("cursor".into(), events, None)); + let records = [ + json!({"step_index":0,"source":"USER_EXPLICIT","type":"USER_INPUT","status":"COMPLETED","content":"List files"}), + json!({"step_index":1,"source":"MODEL","type":"PLANNER_RESPONSE","status":"COMPLETED","content":"Inspecting","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"}}],"usage":{"inputTokens":12,"outputTokens":7,"totalTokens":19}}), + json!({"step_index":2,"source":"MODEL","type":"CORTEX_STEP_TYPE_RUN_COMMAND","status":"COMPLETED","content":"README","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"},"result":"README"}]}), + ]; + let mut contents = String::new(); + let mut boundaries = Vec::new(); + for record in records { + contents.push_str(&serde_json::to_string(&record).unwrap()); + contents.push('\n'); + boundaries.push(contents.len()); } - { - let events = [ - ("sessionStart", json!({})), - ("beforeSubmitPrompt", json!({"prompt":"work"})), - ( - "preToolUse", - json!({"tool_name":"Shell","tool_use_id":"call","tool_input":{"command":"echo hello"}}), - ), - ( - "postToolUse", - json!({"tool_name":"Shell","tool_use_id":"call","tool_output":"hello","tool_input":{"command":"echo hello"},"duration":1}), - ), - ("afterAgentResponse", json!({"text":"done"})), - ("stop", json!({"status":"completed"})), - ("beforeSubmitPrompt", json!({"prompt":"next"})), - ] - .into_iter() - .enumerate() - .map(|(i, (name, mut payload))| { - payload["hook_event_name"] = name.into(); - payload["conversation_id"] = "checkpoint".into(); - payload["generation_id"] = format!("generation-{i}").into(); - payload["model"] = "test".into(); - payload["workspace_roots"] = json!(["/workspace"]); - event("cursor", name, i as i64 + 1, payload) - }) - .collect(); - cases.push(("cursor".to_owned(), events, None)); - } - { - let records = [ - json!({"step_index":0,"source":"USER_EXPLICIT","type":"USER_INPUT","status":"COMPLETED","content":"List files"}), - json!({"step_index":1,"source":"MODEL","type":"PLANNER_RESPONSE","status":"COMPLETED","content":"Inspecting","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"}}],"usage":{"inputTokens":12,"outputTokens":7,"totalTokens":19}}), - json!({"step_index":2,"source":"MODEL","type":"CORTEX_STEP_TYPE_RUN_COMMAND","status":"COMPLETED","content":"README","tool_calls":[{"name":"run_command","args":{"CommandLine":"ls"},"result":"README"}]}), - ]; - let mut contents = String::new(); - let mut boundaries = Vec::new(); - for record in records { - contents.push_str(&serde_json::to_string(&record).unwrap()); - contents.push('\n'); - boundaries.push(contents.len()); - } - let events = [("PreInvocation", boundaries[0]), ("PostInvocation", boundaries[1]), ("PostToolUse", boundaries[2]), ("Stop", boundaries[2])].into_iter().enumerate().map(|(i, (name, through))| { + let events = [("PreInvocation", boundaries[0]), ("PostInvocation", boundaries[1]), ("PostToolUse", boundaries[2]), ("Stop", boundaries[2])].into_iter().enumerate().map(|(i, (name, through))| { event("antigravity", name, i as i64 + 1, json!({"conversationId":"checkpoint","invocationNum":0,"initialNumSteps":1,"stepIdx":2,"fullyIdle":true, "transcriptPath":"/tmp/checkpoint/transcript.jsonl", "_bt_transcript_observation":{"path":"/tmp/checkpoint/transcript.jsonl","observed_bytes":through,"full_path":"/tmp/checkpoint/transcript_full.jsonl","full_observed_bytes":through}, "_bt_transcript_snapshot":{"path":"/tmp/checkpoint/transcript_full.jsonl","contents":contents}})) }).collect(); - cases.push(("antigravity".to_owned(), events, None)); - } - { - let root = - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/grok/transcript"); - let updates = root.join("updates.jsonl"); - let events = root.join("events.jsonl"); - let mut offset = 0; - let stream = std::fs::read(&updates).unwrap().split_inclusive(|byte| *byte == b'\n').map(|line| { + cases.push(("antigravity".to_owned(), events, None)); + let root = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/grok/transcript"); + let updates = root.join("updates.jsonl"); + let events = root.join("events.jsonl"); + let mut offset = 0; + let stream = std::fs::read(&updates).unwrap().split_inclusive(|byte| *byte == b'\n').map(|line| { offset += line.len() as u64; event("grok", "Wake", 1600, json!({"cwd":"/repo","_bt_grok_transcript_mirrors":{"updates":{"mirror":updates,"through":offset},"events":{"mirror":events,"through":std::fs::metadata(&events).unwrap().len()}}})) }).collect(); - cases.push(("grok".to_owned(), stream, None)); - } + cases.push(("grok".to_owned(), stream, None)); cases.push(( "debug".into(), vec![ @@ -157,23 +96,3 @@ pub fn native_streams() -> Vec { )); cases } - -// Include reducer-only correlation fields omitted by public SpanOp JSON. -pub fn comparable(ops: Vec) -> Vec { - let mut rows = ops - .into_iter() - .map(|op| { - let (bt_daemon::SpanOp::Insert(row) | bt_daemon::SpanOp::Merge(row)) = &op; - serde_json::to_string(&( - &op, - &row.turn_span_id, - row.is_turn, - &row.late_merge_key, - &row.late_merge_version, - )) - .unwrap() - }) - .collect::>(); - rows.sort(); - rows -} diff --git a/bt-daemon/tests/translator_checkpoints.rs b/bt-daemon/tests/translator_checkpoints.rs deleted file mode 100644 index 34865d0b..00000000 --- a/bt-daemon/tests/translator_checkpoints.rs +++ /dev/null @@ -1,63 +0,0 @@ -#![allow( - clippy::unwrap_used, - clippy::expect_used, - reason = "Checkpoint regression fixtures fail immediately." -)] -#[path = "support/translation_streams.rs"] -mod streams; -use bt_daemon::wire::Envelope; -use bt_daemon::{AgentTranslator, Registry, SessionCtx, SpanOp}; - -fn collect(translator: &mut dyn AgentTranslator, env: &Envelope, ctx: &SessionCtx) -> Vec { - let mut ops = translator.handle(env, ctx).unwrap(); - while let Some(batch) = translator.drain_pending(ctx).unwrap() { - ops.extend(batch); - } - ops -} -fn assert_stream(source: &str, events: Vec) { - let registry = Registry::default_agents(); - let session = events - .first() - .map(|env| env.session_id.as_str()) - .unwrap_or("checkpoint"); - let ctx = SessionCtx { - session_id: session.into(), - config: None, - }; - for split in 0..=events.len() { - let mut original = registry.create(source, session); - for env in &events[..split] { - collect(original.as_mut(), env, &ctx); - } - let state = original - .save_state() - .unwrap() - .expect("every translator must checkpoint"); - let mut restored = registry.create(source, session); - restored.restore_state(&state).unwrap(); - for env in &events[split..] { - assert_eq!( - streams::comparable(collect(original.as_mut(), env, &ctx)), - streams::comparable(collect(restored.as_mut(), env, &ctx)), - "{source} split {split} event {}", - env.event - ); - } - assert_eq!( - streams::comparable(original.finalize(&ctx).unwrap()), - streams::comparable(restored.finalize(&ctx).unwrap()), - "{source} split {split} finalization" - ); - } -} -#[test] -fn every_translator_preserves_native_state_at_every_event_boundary() { - let mut covered = Vec::new(); - for (source, events, _fixtures) in streams::native_streams() { - covered.push(source.clone()); - assert_stream(&source, events); - } - covered.sort(); - assert_eq!(covered, Registry::default_agents().sources()); -} From 875e34ea767c0c5488c9769c5886f219a1a90782 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 3 Nov 2026 02:26:40 +0800 Subject: [PATCH 16/19] Ensure one translator owns each source session --- bt-daemon/src/server/maintenance.rs | 29 ++++++++++++++ bt-daemon/src/server/mod.rs | 2 + bt-daemon/src/server/sessions.rs | 59 +++++++++++++++-------------- 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 23a2b222..94861e92 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -468,6 +468,35 @@ mod tests { ) } + #[tokio::test] + async fn concurrent_recovery_and_dispatch_share_one_source_producer() { + let f = crate::pipeline::translation::tests::SourceFixture::new("debug").await; + f.source.shutdown().await; + let (daemon, ready) = paused_ingress(&f.options.data_dir); + let start = Arc::new(tokio::sync::Barrier::new(8)); + let mut opening = tokio::task::JoinSet::new(); + for _ in 0..8 { + let daemon = daemon.clone(); + let start = start.clone(); + opening.spawn(async move { + start.wait().await; + daemon + .translation_for(&event("checkpoint", SessionRoute::default())) + .await + .unwrap() + }); + } + let first = opening.join_next().await.unwrap().unwrap(); + while let Some(source) = opening.join_next().await { + assert!( + Arc::ptr_eq(&first, &source.unwrap()), + "source initialization must have one owner" + ); + } + ready.send(()).unwrap(); + drain_all(&daemon).await; + } + #[tokio::test] async fn capture_is_durable_during_startup_and_recovery_stops_before_it() { let tmp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index 4224f12f..ed9ca2d2 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -124,6 +124,7 @@ pub struct Daemon { auth_errors: Mutex>, sessions: Mutex>>, translations: Mutex>>, + translation_init: tokio::sync::Mutex<()>, correlation: Arc, automatic_links: Mutex>, standalone_links: Mutex>, @@ -178,6 +179,7 @@ impl Daemon { auth_errors: Mutex::new(HashMap::new()), sessions: Mutex::new(HashMap::new()), translations: Mutex::new(HashMap::new()), + translation_init: tokio::sync::Mutex::new(()), correlation: Arc::new(crate::correlation::CorrelationRegistry::default()), automatic_links: Mutex::new(HashMap::new()), standalone_links: Mutex::new(HashSet::new()), diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index 1150882b..d2881d7b 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -37,6 +37,16 @@ impl Daemon { &self, env: &Envelope, ) -> anyhow::Result> { + let storage_key = crate::ids::session_namespace(&env.source, &env.session_id); + if let Some(source) = self.translations.lock().get(&storage_key).cloned() { + return Ok(source); + } + // Initialization owns one reducer per source. This lock covers only + // opening its SQLite head and spawning the actor, never capture/replay. + let _init = self.translation_init.lock().await; + if let Some(source) = self.translations.lock().get(&storage_key).cloned() { + return Ok(source); + } let event_path = journal::ensure_source_journal(&self.data_dir, &env.source, &env.session_id).await?; let namespace = @@ -45,38 +55,29 @@ impl Daemon { { env.session_id.clone() } else { - crate::ids::session_namespace(&env.source, &env.session_id) + storage_key.clone() }; - let journal = self + let events = self .journal_writer_for(&env.source, &env.session_id) .await?; - let storage_key = crate::ids::session_namespace(&env.source, &env.session_id); - let existing = self.translations.lock().get(&storage_key).cloned(); - Ok(match existing { - Some(translation) => translation, - None => { - let translation = crate::pipeline::translation::Translation::spawn( - crate::pipeline::translation::TranslationOptions { - source: env.source.clone(), - session_id: env.session_id.clone(), - namespace: namespace.clone(), - version: self.version.clone(), - data_dir: self.data_dir.clone(), - events: journal.clone(), - event_path: event_path.clone(), - input: self.source_input_for(&env.source, &env.session_id), - registry: self.translators.clone(), - correlation: self.correlation.clone(), - correlation_changed: self.correlation_changed.clone(), - }, - ) - .await?; - self.translations - .lock() - .insert(storage_key, translation.clone()); - translation - } - }) + let source = crate::pipeline::translation::Translation::spawn( + crate::pipeline::translation::TranslationOptions { + source: env.source.clone(), + session_id: env.session_id.clone(), + namespace, + version: self.version.clone(), + data_dir: self.data_dir.clone(), + events, + event_path, + input: self.source_input_for(&env.source, &env.session_id), + registry: self.translators.clone(), + correlation: self.correlation.clone(), + correlation_changed: self.correlation_changed.clone(), + }, + ) + .await?; + self.translations.lock().insert(storage_key, source.clone()); + Ok(source) } pub(super) async fn session_for( From c475d125dba21e254fe5f5fb2399ff9386b005f2 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 3 Nov 2026 03:06:14 +0800 Subject: [PATCH 17/19] Limit recovery changes to responsive capture and socket ownership --- bt-daemon/src/journal.rs | 138 +++---- bt-daemon/src/journal/codec.rs | 126 ------- bt-daemon/src/muse/usage.rs | 2 +- bt-daemon/src/pipeline/input.rs | 33 +- bt-daemon/src/pipeline/journal.rs | 132 +------ bt-daemon/src/pipeline/prepare.rs | 13 - bt-daemon/src/pipeline/translation.rs | 476 +++--------------------- bt-daemon/src/server/ingress.rs | 41 +- bt-daemon/src/server/maintenance.rs | 15 +- bt-daemon/src/server/recovery.rs | 118 ------ bt-daemon/src/server/sessions.rs | 11 +- bt-daemon/src/translate/antigravity.rs | 6 - bt-daemon/src/translate/checkpoint.rs | 67 ---- bt-daemon/src/translate/claude.rs | 16 +- bt-daemon/src/translate/codex.rs | 13 +- bt-daemon/src/translate/cursor/mod.rs | 15 +- bt-daemon/src/translate/debug.rs | 2 - bt-daemon/src/translate/grok.rs | 13 +- bt-daemon/src/translate/mod.rs | 24 -- bt-daemon/src/translate/muse.rs | 5 +- bt-daemon/src/translate/muse_events.rs | 2 +- bt-daemon/src/translate/muse_usage.rs | 2 +- bt-daemon/src/translate/opencode.rs | 12 +- bt-daemon/src/translate/pi.rs | 8 +- bt-daemon/src/translate/recent.rs | 4 - bt-daemon/src/translate/tool.rs | 2 +- bt-daemon/src/translate/turn_lineage.rs | 7 - 27 files changed, 158 insertions(+), 1145 deletions(-) delete mode 100644 bt-daemon/src/journal/codec.rs delete mode 100644 bt-daemon/src/translate/checkpoint.rs diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index 25866641..8e0d1a29 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -10,10 +10,8 @@ //! still tell which delivery pipelines a managed child produced after the //! daemon that accepted them has restarted or idle-exited. -mod codec; use crate::pipeline::input::SourcePosition; pub use crate::pipeline::input::{ControlKind, PipelineControl}; -pub(crate) use codec::CodecState; use crate::wire::{BackendAuth, Envelope, RedactedEnvelope, SessionRoute}; use serde::{Deserialize, Serialize}; @@ -226,7 +224,6 @@ pub async fn gc_old_managed_runs(data_dir: &Path, max_age: std::time::Duration) pub struct JournalWriter { file: tokio::fs::File, position: u64, - codec: CodecState, failed_write: bool, } @@ -275,7 +272,6 @@ impl JournalWriter { Ok(Self { file, position, - codec: CodecState::default(), failed_write: false, }) } @@ -332,28 +328,17 @@ impl JournalWriter { #[serde(skip_serializing_if = "Option::is_none")] _bt_capture_sequence: Option, } - let mut redacted = env.redacted(); - let (json, delta) = self.codec.encode(&redacted.payload)?; - let record_type = if let Some(delta) = delta { - redacted.payload = serde_json::to_value(delta)?; - codec::DELTA_RECORD - } else { - "event" - }; + let redacted = env.redacted(); let mut line = serde_json::to_vec(&Captured { envelope: &redacted, - _bt_record_type: record_type, + _bt_record_type: "event", _bt_capture_sequence: sequence, })?; line.push(b'\n'); let before = self.append_bytes(&line).await?; - self.codec.record(self.position, json.clone()); Ok(SourcePosition { before, through: self.position, - decoded_bytes: (line.len() as u64) - .max(json.as_ref().map_or(0, |json| json.len() as u64)), - decoder_base: json, }) } @@ -379,8 +364,6 @@ impl JournalWriter { Ok(SourcePosition { before, through: self.position, - decoded_bytes: line.len() as u64, - decoder_base: None, }) } } @@ -408,8 +391,6 @@ pub struct JournalRecordEntry { pub record: JournalRecord, /// Exclusive byte offset after this record in the journal. pub through: u64, - /// Decoded work, which can be much larger than a compressed record. - pub(crate) decoded_bytes: u64, /// Daemon-local capture order. Kept outside the native event payload. pub(crate) capture_sequence: Option, } @@ -423,7 +404,7 @@ pub struct JournalReader { path: PathBuf, line_no: usize, position: u64, - decoder: CodecState, + legacy_pi: Vec, } impl JournalReader { @@ -431,20 +412,9 @@ impl JournalReader { self.position } - /// The source consumer owns decoding state between recovery reads. - pub(crate) fn swap_decoder_context(&mut self, context: &mut CodecState) { - std::mem::swap(&mut self.decoder, context); - } - - /// Resume any source journal at a committed byte offset with its saved - /// lossless payload cache (and, for old Pi logs, their legacy message base). - pub(crate) async fn resume( - path: &Path, - position: u64, - through: u64, - decoder_context: CodecState, - ) -> anyhow::Result { - decoder_context.validate()?; + /// Seek within new, self-contained records. Legacy delta streams must be + /// read from the beginning to reconstruct their prior message context. + pub(crate) async fn resume(path: &Path, position: u64, through: u64) -> anyhow::Result { let mut file = tokio::fs::File::open(path).await?; anyhow::ensure!( position <= through && through <= file.metadata().await?.len(), @@ -463,7 +433,7 @@ impl JournalReader { path: path.to_owned(), line_no: 0, position, - decoder: decoder_context, + legacy_pi: Vec::new(), }) } /// Open a journal for streaming, reading at most `through` bytes. @@ -484,7 +454,7 @@ impl JournalReader { path: path.to_path_buf(), line_no: 0, position: 0, - decoder: CodecState::default(), + legacy_pi: Vec::new(), })) } @@ -525,7 +495,7 @@ impl JournalReader { let path = self.path.clone(); let line_no = self.line_no; let position = self.position; - let mut decoder = std::mem::take(&mut self.decoder); + let mut decoder = std::mem::take(&mut self.legacy_pi); // Expanding and parsing a large history can be CPU-heavy even // when file reads are async. Never run it on the hook IPC thread. let (decoder, decoded) = tokio::task::spawn_blocking(move || { @@ -533,7 +503,7 @@ impl JournalReader { (decoder, decoded) }) .await?; - self.decoder = decoder; + self.legacy_pi = decoder; if let Some(entry) = decoded? { return Ok(Some(entry)); } @@ -578,7 +548,7 @@ fn decode_record( path: &Path, line_no: usize, position: u64, - decoder: &mut CodecState, + legacy_pi: &mut Vec, ) -> anyhow::Result> { let line = std::str::from_utf8(bytes) .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))? @@ -591,7 +561,6 @@ fn decode_record( let capture_sequence = value .get("_bt_capture_sequence") .and_then(serde_json::Value::as_u64); - let mut decoded_bytes = bytes.len() as u64; let record = if value .get("_bt_record_type") .and_then(serde_json::Value::as_str) @@ -617,30 +586,10 @@ fn decode_record( let mut event: RedactedEnvelope = serde_json::from_value(value) .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))?; match record_type.as_deref() { - Some("event") | Some(codec::DELTA_RECORD) => { - decoder.legacy_pi.clear(); - let json = if record_type.as_deref() == Some(codec::DELTA_RECORD) { - let (payload, json) = decoder.decode(event.payload)?; - decoded_bytes = decoded_bytes.max(json.len() as u64); - event.payload = payload; - Some(json) - } else { - #[derive(Deserialize)] - struct StoredPayload<'a> { - #[serde(borrow)] - payload: &'a serde_json::value::RawValue, - } - let stored: StoredPayload<'_> = serde_json::from_str(line)?; - codec::cache_serialized(stored.payload.get()) - }; - decoder.record(position, json); - } - None => { - decoder.base = None; - expand_legacy_pi_payload(&mut event, &mut decoder.legacy_pi).map_err(|error| { - anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no) - })?; - } + Some("event") => legacy_pi.clear(), + None => expand_legacy_pi_payload(&mut event, legacy_pi).map_err(|error| { + anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no) + })?, Some(kind) => anyhow::bail!("unsupported journal record type {kind:?}"), } JournalRecord::Event(event) @@ -648,7 +597,6 @@ fn decode_record( Ok(Some(JournalRecordEntry { record, through: position, - decoded_bytes, capture_sequence, })) } @@ -836,34 +784,40 @@ mod tests { } #[tokio::test] - async fn delta_recovery_uses_original_json_bytes_not_reader_serialization() { - let tmp = tempfile::tempdir().unwrap(); - let path = tmp.path().join("original-json.ndjson"); - // Key order and number formatting may differ across serde feature - // sets/versions. Dictionaries use the writer's exact JSON. - let raw = format!(r#"{{"z":"{}","a":1e-3}}"#, "x".repeat(4096)); - let first = format!("{{\"_bt_record_type\":\"event\",\"source\":\"debug\",\"session_id\":\"bytes\",\"event\":\"native\",\"ts_ms\":1,\"payload\":{raw}}}\n"); - let mut cache = CodecState::default(); - cache.record(first.len() as u64, Some(raw.clone().into())); - let (_, delta) = cache - .encode(&serde_json::json!({"z":"x".repeat(4096),"a":0.002})) - .unwrap(); - let delta = serde_json::json!({ - "_bt_record_type":codec::DELTA_RECORD, "source":"debug", "session_id":"bytes", "event":"native", "ts_ms":2, - "payload":delta.unwrap() - }); - tokio::fs::write(&path, format!("{first}{delta}\n")) - .await - .unwrap(); - let through = JournalReader::recorded_len(&path).await; + async fn released_pi_deltas_remain_readable_alongside_complete_new_records() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("legacy.ndjson"); + let first = pi_context(vec![serde_json::json!({"role":"user","content":"old"})], 1); + let mut delta = pi_context(Vec::new(), 2); + delta.payload = serde_json::json!({"event":{"_bt_messages_delta":{"common_prefix":1,"suffix":[{"role":"assistant","content":"answer"}]}}}); + tokio::fs::write( + &path, + format!( + "{}\n{}\n", + serde_json::to_string(&first.redacted()).unwrap(), + serde_json::to_string(&delta.redacted()).unwrap() + ), + ) + .await + .unwrap(); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let mut native = first.clone(); + native.payload["event"]["_bt_messages_delta"] = serde_json::json!({"native":"preserve"}); + let through = writer.append(&native).await.unwrap(); let mut reader = JournalReader::open(&path, through).await.unwrap().unwrap(); assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload["a"], - 0.001 + reader.next_entry().await.unwrap().unwrap().payload, + first.payload + ); + let expanded = reader.next_entry().await.unwrap().unwrap(); + assert_eq!( + expanded.payload["event"]["messages"], + serde_json::json!([{"role":"user","content":"old"},{"role":"assistant","content":"answer"}]) + ); + assert_eq!( + reader.next_entry().await.unwrap().unwrap().payload, + native.payload ); - let next = reader.next_entry().await.unwrap().unwrap(); - assert_eq!(next.payload["a"], 0.002); - assert_eq!(next.payload["z"], "x".repeat(4096)); } #[tokio::test] diff --git a/bt-daemon/src/journal/codec.rs b/bt-daemon/src/journal/codec.rs deleted file mode 100644 index cf0b5df6..00000000 --- a/bt-daemon/src/journal/codec.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! Lossless history compression using the last eligible payload as a zstd -//! dictionary. All sources share this storage codec; live events stay intact. -use base64::{engine::general_purpose::STANDARD, Engine}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::sync::Arc; - -const MIN_BYTES: usize = 4 * 1024; -const MAX_BYTES: usize = 8 * 1024 * 1024; -pub(super) const DELTA_RECORD: &str = "payload_zstd_v1"; - -#[derive(Clone, Default, Serialize, Deserialize)] -pub(crate) struct CodecState { - pub(crate) legacy_pi: Vec, - pub(super) base: Option, -} -#[derive(Clone, Serialize, Deserialize)] -pub(super) struct Base { - through: u64, - json: Arc, -} -#[derive(Serialize, Deserialize)] -pub(super) struct Delta { - base: u64, - data: String, -} - -// Cache limits govern retained state, never which event data is journaled. -// Preserve the writer's exact JSON bytes as the compression dictionary. -pub(super) fn cache_serialized(json: &str) -> Option> { - (MIN_BYTES..=MAX_BYTES) - .contains(&json.len()) - .then(|| Arc::from(json)) -} - -impl CodecState { - pub(super) fn encode( - &self, - payload: &Value, - ) -> anyhow::Result<(Option>, Option)> { - let Some(json) = cache_serialized(&serde_json::to_string(payload)?) else { - return Ok((None, None)); - }; - let Some(base) = &self.base else { - return Ok((Some(json), None)); - }; - let mut compressor = zstd::bulk::Compressor::with_dictionary(1, base.json.as_bytes())?; - compressor.set_parameter(zstd::zstd_safe::CParameter::ChecksumFlag(true))?; - let delta = Delta { - base: base.through, - data: STANDARD.encode(compressor.compress(json.as_bytes())?), - }; - let worthwhile = serde_json::to_vec(&delta)?.len() + 16 < json.len(); - Ok((Some(json), worthwhile.then_some(delta))) - } - - pub(super) fn record(&mut self, through: u64, json: Option>) { - // Small intervening hooks need not discard the last history base. - if let Some(json) = json { - self.base = Some(Base { through, json }); - } - } - - pub(super) fn decode(&self, encoded: Value) -> anyhow::Result<(Value, Arc)> { - let delta: Delta = serde_json::from_value(encoded)?; - let base = self - .base - .as_ref() - .filter(|base| base.through == delta.base) - .ok_or_else(|| anyhow::anyhow!("journal dictionary base is unavailable"))?; - anyhow::ensure!( - delta.data.len() <= MAX_BYTES, - "compressed payload exceeds codec budget" - ); - let bytes = zstd::bulk::Decompressor::with_dictionary(base.json.as_bytes())? - .decompress(&STANDARD.decode(delta.data)?, MAX_BYTES)?; - let json = String::from_utf8(bytes)?; - Ok((serde_json::from_str(&json)?, json.into())) - } - - pub(crate) fn validate(&self) -> anyhow::Result<()> { - anyhow::ensure!( - self.base - .as_ref() - .is_none_or(|base| base.json.len() <= MAX_BYTES), - "journal dictionary exceeds codec budget" - ); - Ok(()) - } - pub(crate) fn record_live(&mut self, position: &crate::pipeline::input::SourcePosition) { - self.legacy_pi.clear(); - self.record(position.through, position.decoder_base.clone()); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn dictionary_roundtrip_is_lossless_bounded_and_rejects_invalid_bases() { - let first = json!({"history":"é📚".repeat(1024),"index":1}); - let next = json!({"history":format!("{}more context", "é📚".repeat(1024)),"index":2}); - let mut cache = CodecState::default(); - let (base, delta) = cache.encode(&first).unwrap(); - assert!(delta.is_none()); - cache.record(100, base); - let (_, delta) = cache.encode(&next).unwrap(); - let encoded = serde_json::to_value(delta.unwrap()).unwrap(); - assert!(encoded.to_string().len() < first.to_string().len() / 3); - assert_eq!(cache.decode(encoded.clone()).unwrap().0, next); - assert!(CodecState::default().decode(encoded.clone()).is_err()); - cache.record(101, cache.base.as_ref().map(|base| base.json.clone())); - assert!(cache.decode(encoded.clone()).is_err()); - cache.record(100, Some("different dictionary".into())); - assert!(cache.decode(encoded).is_err()); - for payload in [ - json!({"small":1}), - json!({"large":"x".repeat(MAX_BYTES + 1)}), - ] { - let (base, delta) = cache.encode(&payload).unwrap(); - assert!(base.is_none() && delta.is_none()); - } - } -} diff --git a/bt-daemon/src/muse/usage.rs b/bt-daemon/src/muse/usage.rs index 5b282690..e2af3a67 100644 --- a/bt-daemon/src/muse/usage.rs +++ b/bt-daemon/src/muse/usage.rs @@ -26,7 +26,7 @@ pub(crate) enum UsageScope { Model, } -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Default, Serialize, PartialEq)] pub(crate) struct TokenMetrics { #[serde(skip_serializing_if = "Option::is_none")] pub prompt_tokens: Option, diff --git a/bt-daemon/src/pipeline/input.rs b/bt-daemon/src/pipeline/input.rs index 5bee8698..5aa1b985 100644 --- a/bt-daemon/src/pipeline/input.rs +++ b/bt-daemon/src/pipeline/input.rs @@ -1,5 +1,5 @@ //! Stage input shared by direct capture and recovery. Storage supplies only -//! position/decoder metadata; original live payloads arrive from capture. +//! byte positions; original live payloads arrive from capture. use crate::wire::Envelope; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; @@ -9,14 +9,11 @@ use std::{collections::VecDeque, sync::Arc}; const RECORD_BUDGET: usize = 64; const BYTE_BUDGET: u64 = 8 * 1024 * 1024; -/// Durable input position, returned by the storage side path. Decoder data is -/// used only to checkpoint future recovery, never to construct a live payload. +/// Durable byte range returned by the journal append. #[derive(Clone)] pub(crate) struct SourcePosition { pub before: u64, pub through: u64, - pub decoded_bytes: u64, - pub decoder_base: Option>, } pub(crate) enum SourceEvent { @@ -32,12 +29,7 @@ pub(crate) struct SourceRecord { impl SourceRecord { fn bytes(&self) -> u64 { - self.position.decoded_bytes.saturating_add( - self.position - .decoder_base - .as_ref() - .map_or(0, |json| json.len() as u64), - ) + self.position.through.saturating_sub(self.position.before) } } @@ -121,25 +113,24 @@ mod tests { fn oversized_events_flow_alone_and_eviction_exposes_a_recovery_gap() { let input = SourceInput::default(); let event: Envelope = serde_json::from_value(json!({"source":"debug","session_id":"direct","event":"native","ts_ms":1,"payload":{"history":"x".repeat(9 * 1024 * 1024)}})).unwrap(); - let position = |before, through| SourcePosition { + let stride = 9 * 1024 * 1024; + let position = |before| SourcePosition { before, - through, - decoded_bytes: 9 * 1024 * 1024, - decoder_base: None, + through: before + stride, }; - input.push_event(event.clone(), position(0, 1)); + input.push_event(event.clone(), position(0)); assert!(input.take(0, 0).is_none()); - let SourceEvent::Event(actual) = input.take(0, 1).unwrap().event else { + let SourceEvent::Event(actual) = input.take(0, stride).unwrap().event else { panic!("expected original event"); }; assert_eq!(actual.payload, event.payload); - input.push_event(event.clone(), position(1, 2)); - input.push_event(event, position(2, 3)); + input.push_event(event.clone(), position(stride)); + input.push_event(event, position(2 * stride)); assert!( - input.take(1, 3).is_none(), + input.take(stride, 3 * stride).is_none(), "eviction requires recovery of the gap" ); - assert!(input.take(2, 3).is_some()); + assert!(input.take(2 * stride, 3 * stride).is_some()); assert!(input.is_empty()); } } diff --git a/bt-daemon/src/pipeline/journal.rs b/bt-daemon/src/pipeline/journal.rs index 6c0e237d..a1140c98 100644 --- a/bt-daemon/src/pipeline/journal.rs +++ b/bt-daemon/src/pipeline/journal.rs @@ -49,13 +49,6 @@ pub(crate) struct SpanJournal { payload_reads: Arc, } -#[derive(Clone)] -pub(crate) struct TranslationCheckpoint { - pub input: u64, - pub version: String, - pub state: Vec, -} - /// Explicit codec preserves fields intentionally omitted from wire/plugin JSON. #[derive(Serialize, Deserialize)] struct StoredOp { @@ -195,7 +188,7 @@ impl SpanJournal { // Read the version under the same write transaction as every // upgrade, so another opener cannot run a stale migration plan. let version: u32 = db.query_row("PRAGMA user_version", [], |row| row.get(0))?; - anyhow::ensure!(version <= 7, "unsupported SpanOp journal schema {version}"); + anyhow::ensure!(version <= 6, "unsupported SpanOp journal schema {version}"); if version < 3 { db.execute_batch( "CREATE TABLE IF NOT EXISTS batches ( @@ -245,11 +238,6 @@ impl SpanJournal { prune(&db)?; db.execute_batch("PRAGMA user_version=6")?; } - if version < 7 { - db.execute_batch(" - CREATE TABLE IF NOT EXISTS translation_checkpoint (id INTEGER PRIMARY KEY CHECK(id=1), input INTEGER NOT NULL, version TEXT NOT NULL, state BLOB NOT NULL); - PRAGMA user_version=7")?; - } let tail = db.query_row("SELECT tail FROM publication WHERE id=1", [], |row| row.get(0))?; db.execute_batch("COMMIT")?; Ok((db, tail)) @@ -312,52 +300,6 @@ impl SpanJournal { .await } - pub async fn translation_checkpoint(&self) -> anyhow::Result> { - self.with_db(|db| { - Ok(db - .query_row( - "SELECT input,version,state FROM translation_checkpoint WHERE id=1", - [], - |row| { - Ok(TranslationCheckpoint { - input: row.get(0)?, - version: row.get(1)?, - state: row.get(2)?, - }) - }, - ) - .optional()?) - }) - .await - } - - /// Refresh idle context only at the already committed translation head. - pub async fn save_translation_checkpoint( - &self, - checkpoint: TranslationCheckpoint, - ) -> anyhow::Result<()> { - self.with_db(move |db| { - let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; - let head: u64 = tx.query_row( - "SELECT coalesce(max(input),0) FROM batches WHERE committed=1", - [], - |row| row.get(0), - )?; - anyhow::ensure!( - checkpoint.input == head, - "checkpoint must match the translation head" - ); - tx.execute( - "INSERT OR REPLACE INTO translation_checkpoint VALUES(1,?,?,?)", - params![checkpoint.input, checkpoint.version, checkpoint.state], - )?; - tx.commit()?; - Ok(()) - }) - .await - } - - /// Restart abandons only invisible receipts; a committed identity is never rewritten. pub async fn begin(&self, input: u64, observation: bool) -> anyhow::Result { self.with_db(move |db| { let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; @@ -557,24 +499,14 @@ impl SpanJournal { } pub async fn commit(&self, input: u64) -> anyhow::Result { - self.commit_output(input, None, None).await + self.commit_output(input, None).await } /// Root candidates and translated output share the consumed-input boundary. - #[cfg(test)] pub async fn commit_with_roots( &self, input: u64, roots: &BTreeMap>, - ) -> anyhow::Result { - self.commit_with_context(input, roots, None).await - } - - pub async fn commit_with_context( - &self, - input: u64, - roots: &BTreeMap>, - checkpoint: Option, ) -> anyhow::Result { let roots = roots .iter() @@ -586,7 +518,7 @@ impl SpanJournal { Ok((id.clone(), serde_json::to_vec(&ops)?)) }) .collect::>>()?; - self.commit_output(input, Some(roots), checkpoint).await + self.commit_output(input, Some(roots)).await } pub async fn root_candidates(&self) -> anyhow::Result>> { @@ -616,7 +548,6 @@ impl SpanJournal { &self, input: u64, roots: Option)>>, - checkpoint: Option, ) -> anyhow::Result { let tail = self .with_db(move |db| { @@ -640,13 +571,6 @@ impl SpanJournal { tx.execute("UPDATE batches SET committed=1 WHERE input=?", [input])? == 1, "missing translation batch" ); - if let Some(checkpoint) = checkpoint { - anyhow::ensure!(checkpoint.input == input, "checkpoint input mismatch"); - tx.execute( - "INSERT OR REPLACE INTO translation_checkpoint VALUES(1,?,?,?)", - params![input, checkpoint.version, checkpoint.state], - )?; - } if let Some(roots) = roots { tx.execute("DELETE FROM root_candidates", [])?; for (id, value) in roots { @@ -837,18 +761,6 @@ impl SpanJournal { }).await } - pub async fn has_head(&self, route: &SessionRoute) -> anyhow::Result { - let route = serde_json::to_string(route)?; - self.with_db(move |db| { - Ok(db.query_row( - "SELECT EXISTS(SELECT 1 FROM heads WHERE route=?)", - [route], - |row| row.get(0), - )?) - }) - .await - } - /// A completed subscription can remain dormant until its source has new work. pub async fn route_complete(&self, route: &SessionRoute, input: u64) -> anyhow::Result { let route = route.clone(); @@ -1640,7 +1552,7 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let path = tmp.path().join("concurrent-upgrade.sqlite"); let db = Connection::open(&path).unwrap(); - db.execute_batch("PRAGMA journal_mode=WAL; CREATE TABLE batches(input INTEGER PRIMARY KEY,committed INTEGER NOT NULL DEFAULT 0); PRAGMA user_version=1;").unwrap(); + db.execute_batch("CREATE TABLE batches(input INTEGER PRIMARY KEY,committed INTEGER NOT NULL DEFAULT 0); PRAGMA user_version=1;").unwrap(); drop(db); let ready = Arc::new(tokio::sync::Barrier::new(8)); let mut tasks = Vec::new(); @@ -1658,7 +1570,6 @@ mod tests { } for journal in opened { assert_eq!(journal.tail().await.unwrap(), 0); - assert!(journal.translation_checkpoint().await.unwrap().is_none()); } } @@ -1941,35 +1852,18 @@ mod tests { assert!(roots.process(&vec![root]).await.is_empty()); journal.begin(10, true).await.unwrap(); let db = Connection::open(&path).unwrap(); - let context = TranslationCheckpoint { - input: 10, - version: "test".into(), - state: vec![1, 2], - }; - for table in ["root_candidates", "translation_checkpoint"] { - db.execute_batch(&format!("CREATE TRIGGER fail_state BEFORE INSERT ON {table} BEGIN SELECT RAISE(FAIL,'state write'); END;")).unwrap(); - assert!(journal - .commit_with_context(10, roots.pending(), Some(context.clone())) - .await - .is_err()); - assert_eq!(journal.translated_through().await.unwrap(), 0); - assert!(journal.root_candidates().await.unwrap().is_empty()); - assert!(journal.translation_checkpoint().await.unwrap().is_none()); - db.execute_batch("DROP TRIGGER fail_state").unwrap(); - } + db.execute_batch("CREATE TRIGGER fail_roots BEFORE INSERT ON root_candidates BEGIN SELECT RAISE(FAIL,'root write'); END;").unwrap(); + assert!(journal + .commit_with_roots(10, roots.pending()) + .await + .is_err()); + assert_eq!(journal.translated_through().await.unwrap(), 0); + assert!(journal.root_candidates().await.unwrap().is_empty()); + db.execute_batch("DROP TRIGGER fail_roots").unwrap(); journal - .commit_with_context(10, roots.pending(), Some(context)) + .commit_with_roots(10, roots.pending()) .await .unwrap(); - assert_eq!( - journal - .translation_checkpoint() - .await - .unwrap() - .unwrap() - .input, - 10 - ); journal.close().await.unwrap(); let journal = SpanJournal::open(&path).await.unwrap(); diff --git a/bt-daemon/src/pipeline/prepare.rs b/bt-daemon/src/pipeline/prepare.rs index 76783b53..4148bfc0 100644 --- a/bt-daemon/src/pipeline/prepare.rs +++ b/bt-daemon/src/pipeline/prepare.rs @@ -5,7 +5,6 @@ use crate::{ }; use std::collections::HashSet; -#[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct Prepare { source: String, roots: HashSet, @@ -130,18 +129,6 @@ impl ConfiguredTranslator { } impl crate::translate::AgentTranslator for ConfiguredTranslator { - fn save_state(&self) -> anyhow::Result>> { - let Some(state) = self.inner.save_state()? else { - return Ok(None); - }; - Ok(Some(serde_json::to_vec(&(state, &self.prepare))?)) - } - fn restore_state(&mut self, state: &[u8]) -> anyhow::Result<()> { - let (inner, prepare): (Vec, Prepare) = serde_json::from_slice(state)?; - self.inner.restore_state(&inner)?; - self.prepare = prepare; - Ok(()) - } fn handle( &mut self, env: &crate::wire::Envelope, diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index d6dd575e..3716c3fa 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -1,6 +1,6 @@ //! One ordered source consumer. Output publication is independent of route delivery. use super::input::{SourceEvent, SourceInput, SourcePosition, SourceRecord}; -use super::journal::{SpanJournal, TranslationCheckpoint}; +use super::journal::SpanJournal; use super::prepare::Prepare; use super::roots::Roots; use super::{ @@ -23,35 +23,6 @@ use std::{ }; use tokio::sync::{mpsc, oneshot, Notify}; -const CHECKPOINT_EVENTS: u64 = 64; -const CHECKPOINT_BYTES: u64 = 1024 * 1024; - -#[derive(serde::Serialize, serde::Deserialize)] -pub(crate) struct RecoveryContext { - translator: Box, - pub decoder: crate::journal::CodecState, - capture: Option, - bounds: HashMap, - pub templates: HashMap, -} - -impl RecoveryContext { - pub fn decode( - checkpoint: &TranslationCheckpoint, - version: &str, - through: u64, - ) -> anyhow::Result { - anyhow::ensure!( - checkpoint.version == version && checkpoint.input <= through, - "translation checkpoint is incompatible with the current head" - ); - let decoder = zstd::stream::read::Decoder::new(checkpoint.state.as_slice())?; - let context: Self = serde_json::from_reader(std::io::BufReader::new(decoder))?; - context.decoder.validate()?; - Ok(context) - } -} - #[derive(Clone)] pub(crate) struct Observation { pub key: String, @@ -207,10 +178,6 @@ impl Translation { self.journal.route_settings(route).await } - pub async fn has_head(&self, route: &SessionRoute) -> anyhow::Result { - self.journal.has_head(route).await - } - /// Connect a route to the committed SpanOp stream without exposing its /// storage adapter to the route actor. pub async fn subscribe( @@ -228,7 +195,6 @@ enum State { } struct Publication { input: u64, - decoded_bytes: u64, ordinal: u64, observation: bool, capture: Option, @@ -246,19 +212,14 @@ struct Producer { recovery: Arc>>, translator: Option>, reader: Option, - decoder: crate::journal::CodecState, observers: Vec, origins: WeakOrigins, roots: Roots, capture: Option, pending_snapshots: HashSet, bounds: HashMap, - templates: HashMap, position: u64, committed: u64, - checkpoint_position: u64, - checkpoint_work: u64, - checkpoint_safe: bool, state: State, publication: Option, phase: Boundary, @@ -292,19 +253,14 @@ impl Producer { recovery, translator: None, reader: None, - decoder: Default::default(), observers: Vec::new(), origins: WeakOrigins::default(), roots: Roots::default(), capture: None, pending_snapshots: HashSet::new(), bounds: HashMap::new(), - templates: HashMap::new(), position: 0, committed, - checkpoint_position: 0, - checkpoint_work: 0, - checkpoint_safe: true, state, publication: None, phase: Boundary::Journal, @@ -476,73 +432,8 @@ impl Producer { self.state = State::Recovering(self.failure(error, attempts)); } } - if self.publication.is_none() && self.position == self.committed { - if let Err(error) = self.persist_current_context().await { - tracing::warn!(%error, "translation checkpoint could not be saved at shutdown"); - } - } - } - - async fn checkpoint_context( - &mut self, - input: u64, - ) -> anyhow::Result> { - if !self.checkpoint_safe || self.translator.is_none() { - return Ok(None); - } - let translator = self - .translator - .take() - .ok_or_else(|| anyhow::anyhow!("translator is missing"))?; - let decoder = std::mem::take(&mut self.decoder); - let capture = self.capture.clone(); - let bounds = self.bounds.clone(); - let templates = self.templates.clone(); - let version = self.options.version.clone(); - // Context may include a long active conversation. Its encoding and - // compression must not monopolize a runtime thread serving hook IPC. - let (translator, decoder, checkpoint) = tokio::task::spawn_blocking(move || { - let checkpoint = (|| -> anyhow::Result> { - let Some(state) = translator.save_state()? else { - return Ok(None); - }; - let state = RecoveryContext { - translator: serde_json::value::RawValue::from_string(String::from_utf8( - state, - )?)?, - decoder: decoder.clone(), - capture, - bounds, - templates, - }; - let mut encoder = zstd::stream::Encoder::new(Vec::new(), 1)?; - serde_json::to_writer(&mut encoder, &state)?; - let state = encoder.finish()?; - Ok(Some(TranslationCheckpoint { - input, - version, - state, - })) - })(); - (translator, decoder, checkpoint) - }) - .await?; - self.translator = Some(translator); - self.decoder = decoder; - checkpoint } - async fn persist_current_context(&mut self) -> anyhow::Result<()> { - if self.checkpoint_position == self.committed { - return Ok(()); - } - if let Some(checkpoint) = self.checkpoint_context(self.committed).await? { - self.journal.save_translation_checkpoint(checkpoint).await?; - self.checkpoint_position = self.committed; - self.checkpoint_work = 0; - } - Ok(()) - } fn failure(&self, error: anyhow::Error, attempt: u32) -> Failure { let position = self.committed.saturating_add(1); let mut failure = if matches!(self.phase, Boundary::Translation) { @@ -639,55 +530,6 @@ impl Producer { .await??, ); self.position = 0; - self.decoder = Default::default(); - self.checkpoint_position = 0; - self.checkpoint_work = 0; - self.checkpoint_safe = true; - if let Some(checkpoint) = self.journal.translation_checkpoint().await? { - if checkpoint.version == self.options.version && checkpoint.input <= self.committed { - let restored = async { - let registry = self.options.registry.clone(); - let source = self.options.source.clone(); - let namespace = self.options.namespace.clone(); - let version = self.options.version.clone(); - let committed = self.committed; - let input = checkpoint.input; - let (translator, context) = - tokio::task::spawn_blocking(move || -> anyhow::Result<_> { - let context = - RecoveryContext::decode(&checkpoint, &version, committed)?; - let mut translator = registry - .create_checked_with_session_namespace(&source, &namespace)?; - translator.restore_state(context.translator.get().as_bytes())?; - Ok((translator, context)) - }) - .await??; - let reader = JournalReader::resume( - &self.options.event_path, - input, - self.committed, - Default::default(), - ) - .await?; - self.translator = Some(translator); - self.reader = Some(reader); - self.decoder = context.decoder; - self.capture = context.capture; - self.bounds = context.bounds; - self.templates = context.templates; - self.position = input; - self.checkpoint_position = input; - Ok::<_, anyhow::Error>(()) - } - .await; - match restored { - Ok(()) => return Ok(()), - Err(error) => { - tracing::warn!(%error, "invalid translation checkpoint; rebuilding from journal") - } - } - } - } self.reader = if self.committed > 0 { JournalReader::open(&self.options.event_path, self.committed).await? } else { @@ -698,9 +540,6 @@ impl Producer { async fn next_input(&mut self, through: u64) -> anyhow::Result> { if let Some(record) = self.options.input.take(self.position, through) { - if matches!(&record.event, SourceEvent::Event(_)) { - self.decoder.record_live(&record.position); - } // A future gap resumes from this logical position. The live path // does not open, seek, read, or pass payloads through a journal reader. self.reader = None; @@ -711,13 +550,7 @@ impl Producer { } if self.reader.is_none() { self.reader = Some( - JournalReader::resume( - &self.options.event_path, - self.position, - through, - Default::default(), - ) - .await?, + JournalReader::resume(&self.options.event_path, self.position, through).await?, ); } let reader = self @@ -725,10 +558,7 @@ impl Producer { .as_mut() .ok_or_else(|| anyhow::anyhow!("recovery reader is missing"))?; reader.extend(through); - reader.swap_decoder_context(&mut self.decoder); - let entry = reader.next_record().await; - reader.swap_decoder_context(&mut self.decoder); - let Some(entry) = entry? else { + let Some(entry) = reader.next_record().await? else { return Ok(None); }; let event = match entry.record { @@ -743,8 +573,6 @@ impl Producer { position: SourcePosition { before: self.position, through: entry.through, - decoded_bytes: entry.decoded_bytes, - decoder_base: None, }, })) } @@ -771,33 +599,6 @@ impl Producer { SourceEvent::Event(mut env) => { if let Some(route) = &env.route { self.journal.register(route).await?; - // Discovery needs subscriptions and transcript references, - // not copies of native prompts, responses, or credentials. - let mut template = env.redacted(); - template.payload = env - .payload - .as_object() - .map(|payload| { - serde_json::Value::Object( - payload - .iter() - .filter(|(key, _)| { - matches!( - key.as_str(), - "transcript_path" - | "agent_transcript_path" - | "cwd" - | "_bt_transcript_mirror" - | "_bt_claude_transcript_mirrors" - ) - }) - .map(|(key, value)| (key.clone(), value.clone())) - .collect(), - ) - }) - .unwrap_or_else(|| serde_json::json!({})); - self.templates - .insert(serde_json::to_string(route)?, template); } if crate::translate::canonical_source_name(&env.source) != self.options.source { self.position = input; @@ -841,12 +642,6 @@ impl Producer { self.position = input; continue; } - if matches!(control.kind, ControlKind::Retire) { - // Retirement emits defensive ends without making them - // part of the logical state restored on a later resume. - self.persist_current_context().await?; - self.checkpoint_safe = false; - } self.phase = Boundary::Translation; let ops = self .translate(move |translator, ctx| { @@ -872,7 +667,6 @@ impl Producer { }; self.publication = Some(Publication { input, - decoded_bytes: entry.position.decoded_bytes, ordinal: 0, observation, capture, @@ -972,40 +766,15 @@ impl Producer { pending.complete = pending.ops.is_none(); pending.prepared = false; } - // Charge at least 1/64 of the budget per record, bounding both - // event count and decoded bytes with one replay-work counter. - let work = self.checkpoint_work.saturating_add( - pending - .decoded_bytes - .max(pending.input.saturating_sub(self.position)) - .max(CHECKPOINT_BYTES / CHECKPOINT_EVENTS), - ); if publish { self.phase = Boundary::Journal; - let checkpoint = if work >= CHECKPOINT_BYTES { - self.checkpoint_context(pending.input).await? - } else { - None - }; - let checkpointed = checkpoint.is_some(); self.journal - .commit_with_context(pending.input, self.roots.pending(), checkpoint) + .commit_with_roots(pending.input, self.roots.pending()) .await?; - if checkpointed { - self.checkpoint_position = pending.input; - self.checkpoint_work = 0; - } else { - self.checkpoint_work = work; - } self.committed = pending.input; self.progress .committed .store(pending.input, Ordering::Relaxed); - } else { - // Restoring a checkpoint replays the committed suffix too. - // Count that work so repeated short-lived processes cannot - // keep extending the distance from the same old checkpoint. - self.checkpoint_work = work; } self.position = pending.input; Ok(()) @@ -1091,13 +860,21 @@ pub(crate) mod tests { pub(crate) async fn new(agent: &str) -> Self { let dir = tempfile::tempdir().unwrap(); let path = crate::journal::source_journal_path(dir.path(), agent, "checkpoint"); - let options = resume_options( - dir.path(), - agent, - Arc::new(tokio::sync::Mutex::new( + let options = TranslationOptions { + source: agent.into(), + session_id: "checkpoint".into(), + namespace: crate::ids::session_namespace(agent, "checkpoint"), + version: "test".into(), + data_dir: dir.path().into(), + events: Arc::new(tokio::sync::Mutex::new( JournalWriter::open_path(&path).await.unwrap(), )), - ); + event_path: path, + input: SourceInput::default(), + registry: Arc::new(Registry::default_agents()), + correlation: Arc::default(), + correlation_changed: Arc::default(), + }; let source = Translation::spawn(options.clone()).await.unwrap(); Self { _dir: dir, @@ -1130,14 +907,6 @@ pub(crate) mod tests { self.options.input = SourceInput::default(); self.source = Translation::spawn(self.options.clone()).await.unwrap(); } - async fn checkpoint(&self) -> TranslationCheckpoint { - self.source - .journal - .translation_checkpoint() - .await - .unwrap() - .unwrap() - } fn db(&self) -> rusqlite::Connection { rusqlite::Connection::open(self.options.data_dir.join("pipeline").join(format!( "{}.sqlite", @@ -1169,9 +938,6 @@ pub(crate) mod tests { } let mut cases = vec![("live", 0), ("overflow", 2)]; cases.extend((1..=events.len()).map(|split| ("cold", split))); - if agent == "debug" { - cases.extend([("old", 1), ("corrupt", 1)]); - } let mut expected = None; for (mode, split) in cases { let mut f = SourceFixture::new(&agent).await; @@ -1181,21 +947,15 @@ pub(crate) mod tests { } let tail = *positions.last().unwrap(); if mode == "cold" && split == 1 { - let mut reader = JournalReader::open(&f.options.event_path, tail) + let stored = tokio::fs::read_to_string(&f.options.event_path) .await - .unwrap() .unwrap(); - for event in &events { - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - event.payload - ); + assert_eq!(stored.lines().count(), events.len()); + for (line, event) in stored.lines().zip(&events) { + let record: serde_json::Value = serde_json::from_str(line).unwrap(); + assert_eq!(record["payload"], event.payload); } } - assert!(tokio::fs::read_to_string(&f.options.event_path) - .await - .unwrap() - .contains("payload_zstd_v1")); match mode { "live" => { f.process(positions[1]).await; @@ -1215,17 +975,6 @@ pub(crate) mod tests { _ => { f.process(positions[split]).await; f.source.shutdown().await; - if mode == "cold" { - f.poison(0..positions[split]).await; - } else { - f.db() - .execute_batch(if mode == "old" { - "UPDATE translation_checkpoint SET version='old'" - } else { - "UPDATE translation_checkpoint SET state=x'00'" - }) - .unwrap(); - } f.reopen().await; } } @@ -1255,7 +1004,6 @@ pub(crate) mod tests { } struct BlockedTranslator { - checkpoint: bool, entered: Arc, release: std::sync::mpsc::Receiver<()>, } @@ -1273,169 +1021,50 @@ pub(crate) mod tests { _: &crate::wire::Envelope, _: &SessionCtx, ) -> anyhow::Result> { - if !self.checkpoint { - self.block(); - } + self.block(); Ok(Vec::new()) } - fn save_state(&self) -> anyhow::Result>> { - if self.checkpoint { - self.block(); - } - Ok(Some(b"{}".to_vec())) - } - } - #[tokio::test] - async fn synchronous_replay_and_checkpoint_work_do_not_block_capture() { - for checkpoint in [false, true] { - let f = SourceFixture::new("debug").await; - let event = streams::native_streams().pop().unwrap().1.remove(0); - let through = f.write(&event, false).await; - let mut producer = Producer::open(f.options.clone()).await.unwrap(); - let entered = Arc::new(Notify::new()); - let (release, receiver) = std::sync::mpsc::channel(); - producer.translator = Some(Box::new(BlockedTranslator { - checkpoint, - entered: entered.clone(), - release: receiver, - })); - let started = std::time::Instant::now(); - let work = tokio::spawn(async move { - let result = if checkpoint { - producer.checkpoint_context(0).await.map(|_| ()) - } else { - producer.advance(through).await - }; - (producer, result) - }); - entered.notified().await; - assert!(started.elapsed() < std::time::Duration::from_secs(1)); - assert!( - tokio::time::timeout(std::time::Duration::from_secs(1), f.write(&event, true)) - .await - .unwrap() - > through - ); - assert!( - !work.is_finished(), - "capture must complete before replay is released" - ); - release.send(()).unwrap(); - let (producer, result) = work.await.unwrap(); - result.unwrap(); - assert_eq!( - producer.journal.translated_through().await.unwrap(), - if checkpoint { 0 } else { through } - ); - f.source.shutdown().await; - } } #[tokio::test] - async fn checkpoint_budget_counts_reconstructed_inputs_and_decoded_bytes() { - let mut f = SourceFixture::new("debug").await; - let mut event = streams::native_streams().pop().unwrap().1.remove(0); - let mut tail = 0; - for _ in 0..2 * CHECKPOINT_EVENTS - 1 { - tail = f.write(&event, false).await; - } - f.process(tail).await; - assert!(f.checkpoint().await.input < tail); - let task = f.source.task.lock().await.take().unwrap(); - task.abort(); - let _ = task.await; - tail = f.write(&event, false).await; - f.reopen().await; - f.process(tail).await; - assert_eq!( - f.checkpoint().await.input, - tail, - "reconstructed suffix counts toward the next checkpoint" - ); - let before = tail; - event.payload["history"] = "x".repeat(256 * 1024).into(); - for _ in 0..5 { - tail = f.write(&event, false).await; - } + async fn synchronous_replay_does_not_block_capture_or_chase_new_events() { + let f = SourceFixture::new("debug").await; + let event = streams::native_streams().pop().unwrap().1.remove(0); + let through = f.write(&event, false).await; + let mut producer = Producer::open(f.options.clone()).await.unwrap(); + let entered = Arc::new(Notify::new()); + let (release, receiver) = std::sync::mpsc::channel(); + producer.translator = Some(Box::new(BlockedTranslator { + entered: entered.clone(), + release: receiver, + })); + let started = std::time::Instant::now(); + let work = tokio::spawn(async move { + let result = producer.advance(through).await; + (producer, result) + }); + entered.notified().await; + assert!(started.elapsed() < std::time::Duration::from_secs(1)); assert!( - tail - before < CHECKPOINT_BYTES, - "stored deltas must be below the physical byte threshold" + tokio::time::timeout(std::time::Duration::from_secs(1), f.write(&event, true)) + .await + .unwrap() + > through ); - f.process(tail).await; assert!( - f.checkpoint().await.input > before, - "decoded work must trigger a checkpoint before 64 inputs" + !work.is_finished(), + "capture must complete before replay is released" ); - f.source.shutdown().await; - } - - #[tokio::test] - async fn legacy_pi_decoder_context_resumes_without_reading_its_prefix() { - let mut f = SourceFixture::new("pi").await; - let mut event = streams::native_streams().pop().unwrap().1.remove(0); - event.source = "pi".into(); - event.event = "context".into(); - let mut contents = String::new(); - let mut positions = Vec::new(); - for i in 0..CHECKPOINT_EVENTS + 3 { - event.payload = serde_json::json!({"event":{"_bt_messages_delta":{"common_prefix":if i == 0 {0} else {1}, "suffix":[{"role":"user","content":"shared".repeat(1024)}]}}}); - contents.push_str(&serde_json::to_string(&event.redacted()).unwrap()); - contents.push('\n'); - positions.push(contents.len() as u64); - } - tokio::fs::write(&f.options.event_path, contents) - .await - .unwrap(); - f.reopen().await; - let tail = *positions.last().unwrap(); - f.process(tail).await; - let saved = f.checkpoint().await; - assert!(saved.input < tail); - assert!(!RecoveryContext::decode(&saved, "test", tail) - .unwrap() - .decoder - .legacy_pi - .is_empty()); - let task = f.source.task.lock().await.take().unwrap(); - task.abort(); - let _ = task.await; - f.poison(0..saved.input).await; - f.reopen().await; - f.process(tail).await; - event.payload = serde_json::json!({"event":{"messages":[{"role":"user","content":"native"}],"_bt_messages_delta":{"native_field":"preserve"}}}); - let native = f.write(&event, false).await; - f.process(native).await; - let mut reader = - JournalReader::resume(&f.options.event_path, tail, native, Default::default()) - .await - .unwrap(); + release.send(()).unwrap(); + let (producer, result) = work.await.unwrap(); + result.unwrap(); assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - event.payload + producer.journal.translated_through().await.unwrap(), + through ); f.source.shutdown().await; } - fn resume_options( - data_dir: &std::path::Path, - source: &str, - events: Arc>, - ) -> TranslationOptions { - TranslationOptions { - source: source.into(), - session_id: "checkpoint".into(), - namespace: crate::ids::session_namespace(source, "checkpoint"), - version: "test".into(), - data_dir: data_dir.into(), - events, - event_path: crate::journal::source_journal_path(data_dir, source, "checkpoint"), - input: SourceInput::default(), - registry: Arc::new(Registry::default_agents()), - correlation: Arc::default(), - correlation_changed: Arc::default(), - } - } - #[tokio::test] async fn shared_storage_stays_open_until_the_last_route_finishes_draining() { let f = SourceFixture::new("debug").await; @@ -1595,7 +1224,6 @@ pub(crate) mod tests { producer.translator = Some(Box::new(CountedTranslation(drains.clone()))); producer.publication = Some(Publication { input: 10, - decoded_bytes: 0, ordinal: 0, observation: true, capture: None, diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index fece76df..55e16fe8 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -60,53 +60,28 @@ impl Daemon { } pub(super) async fn drain_capture_buffer(daemon: &Arc) -> anyhow::Result<()> { - let mut cursors = HashMap::new(); loop { match daemon.next_capture_work() { - CaptureWork::Evicted(bounds) => replay_evicted(daemon, &mut cursors, &bounds).await?, + CaptureWork::Evicted(bounds) => replay_evicted(daemon, &bounds).await?, CaptureWork::Event(event) => dispatch_ingress_event(daemon, *event).await, CaptureWork::Finished => return Ok(()), } } } -/// Resume frozen evicted ranges, merging one lookahead per source in capture -/// order. Retain readers between advances so another eviction can extend them. +/// Read frozen evicted ranges, merging one lookahead per source in capture +/// order. Current captures contain complete payloads and need no prefix replay. pub(super) async fn replay_evicted( daemon: &Arc, - cursors: &mut HashMap, bounds: &HashMap, ) -> anyhow::Result<()> { let mut ready = Vec::new(); for (path, range) in bounds { - let mut reader = match cursors.remove(path) { - Some(reader) => reader, - None => match resume_discovery(daemon, path, range.from).await { - Ok(Some((reader, _))) => reader, - _ => journal::JournalReader::open(path, range.from) - .await? - .ok_or_else(|| anyhow::anyhow!("evicted capture journal disappeared"))?, - }, - }; - anyhow::ensure!( - reader.position() <= range.from, - "evicted capture ranges overlap" - ); - // Decode only to reconstruct the payload cache across a gap whose - // events were already dispatched from memory. - reader.extend(range.from); - while reader.next_record().await?.is_some() {} - anyhow::ensure!( - reader.position() == range.from, - "invalid evicted capture boundary" - ); - reader.extend(range.through); + let mut reader = journal::JournalReader::resume(path, range.from, range.through).await?; if let Some((sequence, event)) = next_evicted_event(daemon, &mut reader, range.through).await? { - ready.push((sequence, event, path.clone(), reader, range.through)); - } else { - cursors.insert(path.clone(), reader); + ready.push((sequence, event, reader, range.through)); } } while let Some(index) = ready @@ -115,12 +90,10 @@ pub(super) async fn replay_evicted( .min_by_key(|(_, item)| item.0) .map(|(index, _)| index) { - let (_, event, path, mut reader, through) = ready.swap_remove(index); + let (_, event, mut reader, through) = ready.swap_remove(index); dispatch_ingress_event(daemon, event).await; if let Some((sequence, event)) = next_evicted_event(daemon, &mut reader, through).await? { - ready.push((sequence, event, path, reader, through)); - } else { - cursors.insert(path, reader); + ready.push((sequence, event, reader, through)); } } Ok(()) diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 94861e92..3de48e0d 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -548,7 +548,7 @@ mod tests { } #[tokio::test] - async fn repeated_overflow_preserves_each_session_and_its_compressed_context() { + async fn repeated_overflow_preserves_each_session_and_event_order() { let tmp = tempfile::tempdir().unwrap(); let (daemon, ready) = paused_ingress(tmp.path()); // Four journals interleave through two eviction ranges and a live tail. @@ -584,25 +584,16 @@ mod tests { total - INGRESS_QUEUE_CAPACITY ); } - let mut cursors = HashMap::new(); - replay_evicted(&daemon, &mut cursors, &first.unwrap()) - .await - .unwrap(); + replay_evicted(&daemon, &first.unwrap()).await.unwrap(); let CaptureWork::Evicted(second) = daemon.next_capture_work() else { panic!("new captures must extend recovery"); }; - replay_evicted(&daemon, &mut cursors, &second) - .await - .unwrap(); + replay_evicted(&daemon, &second).await.unwrap(); drain_capture_buffer(&daemon).await.unwrap(); ready.send(()).unwrap(); daemon.settle_ingress().await; for (slot, env) in envs.iter().enumerate() { let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); - assert!(tokio::fs::read_to_string(&path) - .await - .unwrap() - .contains("payload_zstd_v1")); let storage = tmp.path().join("pipeline").join(format!( "{}.sqlite", crate::ids::session_storage_id(&env.source, &env.session_id) diff --git a/bt-daemon/src/server/recovery.rs b/bt-daemon/src/server/recovery.rs index 348a7561..1166fa8e 100644 --- a/bt-daemon/src/server/recovery.rs +++ b/bt-daemon/src/server/recovery.rs @@ -2,59 +2,6 @@ use super::*; -/// Checkpoint templates replace the completed prefix scan. Legacy journals -/// and incompatible checkpoints retain the full-log recovery path. -pub(super) async fn resume_discovery( - daemon: &Daemon, - path: &std::path::Path, - through: u64, -) -> anyhow::Result)>> { - let Some((_, id)) = path - .file_stem() - .and_then(|name| name.to_str()) - .and_then(|name| name.rsplit_once("--")) - else { - return Ok(None); - }; - if uuid::Uuid::parse_str(id).is_err() { - return Ok(None); - } - let storage = daemon - .data_dir - .join("pipeline") - .join(format!("{id}.sqlite")); - if !tokio::fs::try_exists(&storage).await? { - return Ok(None); - } - let partition = crate::pipeline::journal::SpanJournal::open(&storage).await?; - let Some(checkpoint) = partition.translation_checkpoint().await? else { - return Ok(None); - }; - let head = through.min(partition.translated_through().await?); - let version = daemon.version.clone(); - let input = checkpoint.input; - let context = tokio::task::spawn_blocking(move || { - crate::pipeline::translation::RecoveryContext::decode(&checkpoint, &version, head) - }) - .await??; - let Some(env) = context.templates.values().next() else { - return Ok(None); - }; - anyhow::ensure!( - path == journal::source_journal_path(&daemon.data_dir, &env.source, &env.session_id), - "checkpoint does not belong to this journal" - ); - anyhow::ensure!( - context - .templates - .values() - .all(|template| template.source == env.source && template.session_id == env.session_id), - "checkpoint contains another source session" - ); - let reader = journal::JournalReader::resume(path, input, through, context.decoder).await?; - Ok(Some((reader, context.templates.into_values().collect()))) -} - pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec { let pending: Vec = daemon .pending_sessions @@ -87,32 +34,6 @@ pub(super) async fn discover_unprocessed_journals(daemon: &Arc) -> Vec

= HashMap::new(); let mut latest_by_route: HashMap = HashMap::new(); - match resume_discovery(daemon, &path, recorded_len).await { - Ok(Some((resumed, templates))) => { - reader = resumed; - let input = reader.position(); - before = input; - for template in templates { - let env = journal::envelope_from_redacted(template); - daemon.remember_claude_transcripts(&env); - if let Some(route) = &env.route { - latest_by_route.insert( - serde_json::to_string(route).unwrap_or_default(), - PendingEvent { - env, - replay_through: input, - journal_through: input, - settings_through: Some(input), - }, - ); - } - } - } - Ok(None) => {} - Err(error) => { - tracing::warn!(%error, path=%path.display(), "source discovery checkpoint unavailable; scanning journal") - } - } while let Ok(Some(entry)) = reader.next_record().await { let through = entry.through; match entry.record { @@ -324,45 +245,6 @@ pub(super) mod tests { } } - #[tokio::test] - async fn startup_discovery_uses_the_session_checkpoint_instead_of_scanning_history() { - use crate::pipeline::translation::tests::SourceFixture; - let f = SourceFixture::new("debug").await; - let mut env: Envelope = serde_json::from_value(serde_json::json!({ - "source":"debug", "session_id":"checkpoint", "event":"old", "ts_ms":1, - "payload":{}, "route":SessionRoute::default(), - })) - .unwrap(); - let mut through = 0; - for i in 0..64 { - env.ts_ms = i; - through = f.write(&env, false).await; - } - f.process(through).await; - f.source.shutdown().await; - f.poison(0..through).await; - env.event = "after".into(); - let tail = f.write(&env, false).await; - let raw = &f.options.event_path; - let daemon = Daemon::new( - debug_serve_options("test", &f.options.data_dir), - f.options.data_dir.clone(), - ); - let (reader, _) = resume_discovery(&daemon, raw, tail).await.unwrap().unwrap(); - assert_eq!(reader.position(), through); - let candidates = discover_unprocessed_journals(&daemon).await; - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].env.event, "after"); - assert_eq!(candidates[0].journal_through, tail); - for event in candidates { - dispatch_ingress_event(&daemon, event).await; - } - let source = daemon.translations.lock().values().next().unwrap().clone(); - source.barrier().await.unwrap(); - assert!(source.error.lock().is_none()); - super::super::maintenance::drain_all(&daemon).await; - } - #[tokio::test] async fn pending_capture_does_not_suppress_an_unrelated_durable_head() { let tmp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index d2881d7b..2c63f913 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -107,15 +107,10 @@ impl Daemon { // delivery checkpoint controlling which rows reach its sink. let journal_path = journal::ensure_source_journal(&self.data_dir, &env.source, &env.session_id).await?; - let translation = self.translation_for(env).await?; - // Existing SQLite heads are authoritative; subscribe never reseeds - // them from legacy checkpoints. Avoid decoding the entire old prefix. - let legacy_through = if translation.has_head(&route).await? { - 0 - } else { + let legacy_through = journal::JournalReader::acknowledged_through(&journal_path, replay_through, &route) - .await - }; + .await; + let translation = self.translation_for(env).await?; let settings = translation.route_settings(&route).await?; let config = settings.with_auth(config.auth); env.config = Some(config.clone()); diff --git a/bt-daemon/src/translate/antigravity.rs b/bt-daemon/src/translate/antigravity.rs index 65305823..d40d5a24 100644 --- a/bt-daemon/src/translate/antigravity.rs +++ b/bt-daemon/src/translate/antigravity.rs @@ -27,7 +27,6 @@ impl TranslatorFactory for crate::agents::Antigravity { } } -#[derive(serde::Serialize, serde::Deserialize)] struct Turn { span_id: String, number: u32, @@ -35,7 +34,6 @@ struct Turn { last_output: Option, } -#[derive(serde::Serialize, serde::Deserialize)] struct Invocation { span_id: String, parent_span_id: String, @@ -43,14 +41,12 @@ struct Invocation { record_start: usize, } -#[derive(serde::Serialize, serde::Deserialize)] struct PendingTool { span_id: String, parent_span_id: String, name: String, } -#[derive(serde::Serialize, serde::Deserialize)] struct AntigravityTranslator { session_id: String, session_span_id: String, @@ -67,7 +63,6 @@ struct AntigravityTranslator { invocations: HashMap, tools: HashMap, last_ts_ms: i64, - #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, } @@ -499,7 +494,6 @@ impl AntigravityTranslator { } impl AgentTranslator for AntigravityTranslator { - translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); let mut ops = Vec::new(); diff --git a/bt-daemon/src/translate/checkpoint.rs b/bt-daemon/src/translate/checkpoint.rs deleted file mode 100644 index 8e60c2ae..00000000 --- a/bt-daemon/src/translate/checkpoint.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Checkpoint-only SpanRow codec. The public row JSON intentionally omits -//! internal correlation and merge fields; reducer checkpoints must retain them. -use super::{LateMergeVersion, SpanRow}; -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize)] -struct Row { - value: SpanRow, - late_merge_key: Option, - late_merge_version: Option, - turn_span_id: Option, - is_turn: bool, -} -impl Row { - fn from_row(value: &SpanRow) -> Self { - Self { - value: value.clone(), - late_merge_key: value.late_merge_key.clone(), - late_merge_version: value.late_merge_version.clone(), - turn_span_id: value.turn_span_id.clone(), - is_turn: value.is_turn, - } - } - fn into_row(mut self) -> SpanRow { - self.value.late_merge_key = self.late_merge_key; - self.value.late_merge_version = self.late_merge_version; - self.value.turn_span_id = self.turn_span_id; - self.value.is_turn = self.is_turn; - self.value - } -} -pub(super) mod row { - use super::*; - pub fn serialize( - value: &SpanRow, - serializer: S, - ) -> Result { - Row::from_row(value).serialize(serializer) - } - pub fn deserialize<'de, D: serde::Deserializer<'de>>( - deserializer: D, - ) -> Result { - Ok(Row::deserialize(deserializer)?.into_row()) - } -} -pub(super) mod rows { - use super::*; - use std::collections::BTreeMap; - pub fn serialize( - value: &BTreeMap, - serializer: S, - ) -> Result { - value - .iter() - .map(|(id, row)| (id, Row::from_row(row))) - .collect::>() - .serialize(serializer) - } - pub fn deserialize<'de, D: serde::Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> { - Ok(BTreeMap::::deserialize(deserializer)? - .into_iter() - .map(|(id, row)| (id, row.into_row())) - .collect()) - } -} diff --git a/bt-daemon/src/translate/claude.rs b/bt-daemon/src/translate/claude.rs index 2a2ca8de..2fdcb217 100644 --- a/bt-daemon/src/translate/claude.rs +++ b/bt-daemon/src/translate/claude.rs @@ -140,7 +140,6 @@ impl TranslatorFactory for crate::agents::Claude { } } -#[derive(serde::Serialize, serde::Deserialize)] struct Turn { id: String, number: u32, @@ -150,32 +149,29 @@ struct Turn { effort: Option, } -#[derive(serde::Serialize, serde::Deserialize)] struct PendingPrompt { turn_id: Arc, content: Value, } -#[derive(Default, serde::Serialize, serde::Deserialize)] +#[derive(Default)] struct TranscriptCursor { offset: u64, buffered: Vec, } -#[derive(serde::Serialize, serde::Deserialize)] struct Subagent { span_id: String, parent_span_id: String, transcript_path: Option, } -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone)] struct AgentOrigin { parent_id: String, user_turn_id: Option, } -#[derive(serde::Serialize, serde::Deserialize)] struct PendingTool { span_id: String, parent_id: String, @@ -183,20 +179,18 @@ struct PendingTool { metadata: Value, } -#[derive(serde::Serialize, serde::Deserialize)] enum PendingHistory { Main, Owned(MessageHistory), } -#[derive(Default, serde::Serialize, serde::Deserialize)] +#[derive(Default)] struct MessageHistory { messages: Vec, preserved_after_compaction: Vec, fork_context_prefix: bool, } -#[derive(serde::Serialize, serde::Deserialize)] struct HistoryMessage { value: Value, source_uuids: Vec, @@ -262,7 +256,6 @@ impl MessageHistory { } } -#[derive(serde::Serialize, serde::Deserialize)] struct PendingEmission { segments: VecDeque>, history: PendingHistory, @@ -272,7 +265,6 @@ struct PendingEmission { clear_after: bool, } -#[derive(serde::Serialize, serde::Deserialize)] struct ClaudeTranslator { session_id: String, session_span_id: String, @@ -314,7 +306,6 @@ struct ClaudeTranslator { permission_mode: Option, effort: Option, system_prompt: Option, - #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, current_cwd: Option, last_turn_cwd: Option, @@ -1378,7 +1369,6 @@ impl ClaudeTranslator { } impl AgentTranslator for ClaudeTranslator { - translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); anyhow::ensure!( diff --git a/bt-daemon/src/translate/codex.rs b/bt-daemon/src/translate/codex.rs index be91fcdd..f6731819 100644 --- a/bt-daemon/src/translate/codex.rs +++ b/bt-daemon/src/translate/codex.rs @@ -139,13 +139,12 @@ impl TranslatorFactory for crate::agents::Codex { } } -#[derive(PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)] +#[derive(PartialEq, Eq, Clone, Copy)] enum ScopeKind { Main, Subagent, } -#[derive(serde::Serialize, serde::Deserialize)] struct OpenTurn { turn_id: String, span_id: String, @@ -157,13 +156,12 @@ struct OpenTurn { native_prompt_seen: bool, } -#[derive(PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)] +#[derive(PartialEq, Eq, Clone, Copy)] enum TurnInputSource { Native, Authoritative, } -#[derive(serde::Serialize, serde::Deserialize)] struct OpenLlm { span_id: String, turn_id: String, @@ -173,7 +171,6 @@ struct OpenLlm { output_preset: bool, } -#[derive(serde::Serialize, serde::Deserialize)] struct OpenTool { span_id: String, turn_id: String, @@ -181,7 +178,6 @@ struct OpenTool { name: String, } -#[derive(serde::Serialize, serde::Deserialize)] struct Scope { path: String, kind: ScopeKind, @@ -216,7 +212,6 @@ impl Scope { } } -#[derive(serde::Serialize, serde::Deserialize)] enum DeferredHook { None, PostToolUse(Value), @@ -235,7 +230,6 @@ enum DeferredHook { }, } -#[derive(serde::Serialize, serde::Deserialize)] enum PendingWork { Hook { path: String, @@ -252,7 +246,6 @@ enum PendingWork { }, } -#[derive(serde::Serialize, serde::Deserialize)] struct CodexTranslator { session_id: String, /// The daemon-owned session row. Child spans always parent to this id. @@ -274,12 +267,10 @@ struct CodexTranslator { compaction_trigger_by_turn: RecentMap, compaction_spans: RecentSet, pending: Option, - #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, } impl AgentTranslator for CodexTranslator { - translator_checkpoint!(); fn handle(&mut self, event: &Envelope, _ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), diff --git a/bt-daemon/src/translate/cursor/mod.rs b/bt-daemon/src/translate/cursor/mod.rs index e8b26958..18b59369 100644 --- a/bt-daemon/src/translate/cursor/mod.rs +++ b/bt-daemon/src/translate/cursor/mod.rs @@ -73,7 +73,7 @@ impl TranslatorFactory for crate::agents::Cursor { } } -#[derive(Default, serde::Serialize, serde::Deserialize)] +#[derive(Default)] struct History { messages: VecDeque, bytes: usize, @@ -100,7 +100,7 @@ impl History { } } -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone)] struct Turn { ordinal: u64, id: String, @@ -113,9 +113,7 @@ struct Turn { emitted_text: String, usage: serde_json::Map, } -#[derive(serde::Serialize, serde::Deserialize)] struct ModelStep { - #[serde(with = "crate::translate::checkpoint::row")] row: SpanRow, content: Vec, text: String, @@ -155,15 +153,14 @@ impl ModelStep { self.content.push(value); } } -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone)] struct Tool { - #[serde(with = "crate::translate::checkpoint::row")] row: SpanRow, turn: String, call_id: String, ended: bool, } -#[derive(Default, serde::Serialize, serde::Deserialize)] +#[derive(Default)] struct TranscriptCursor { path: String, offset: u64, @@ -211,7 +208,6 @@ impl TranscriptCursor { } } -#[derive(serde::Serialize, serde::Deserialize)] struct CursorTranslator { namespace: String, root: String, @@ -222,7 +218,6 @@ struct CursorTranslator { cwd: Option, model: Option, model_params: Option, - #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, turn_seq: u64, turn: Option, @@ -238,7 +233,6 @@ struct CursorTranslator { transcript_owner: Option, transcript_unattributed: bool, pending: Option, - #[serde(with = "crate::translate::checkpoint::rows")] subagents: BTreeMap, compact_seq: u64, } @@ -1541,7 +1535,6 @@ impl CursorTranslator { } } impl AgentTranslator for CursorTranslator { - translator_checkpoint!(); fn handle(&mut self, e: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), diff --git a/bt-daemon/src/translate/debug.rs b/bt-daemon/src/translate/debug.rs index 5e9f2555..6af1a0b5 100644 --- a/bt-daemon/src/translate/debug.rs +++ b/bt-daemon/src/translate/debug.rs @@ -36,7 +36,6 @@ impl TranslatorFactory for DebugTranslatorFactory { } } -#[derive(serde::Serialize, serde::Deserialize)] struct DebugTranslator { root_span_id: String, root_emitted: bool, @@ -44,7 +43,6 @@ struct DebugTranslator { } impl AgentTranslator for DebugTranslator { - translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { let mut ops = Vec::new(); diff --git a/bt-daemon/src/translate/grok.rs b/bt-daemon/src/translate/grok.rs index cc9adbd7..d474844c 100644 --- a/bt-daemon/src/translate/grok.rs +++ b/bt-daemon/src/translate/grok.rs @@ -32,7 +32,7 @@ impl TranslatorFactory for crate::agents::Grok { } } -#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Default)] struct BoundedOutput { values: Vec, bytes: usize, @@ -111,7 +111,6 @@ impl BoundedOutput { } } -#[derive(serde::Serialize, serde::Deserialize)] struct OpenTurn { span_id: String, key: String, @@ -129,7 +128,6 @@ struct TurnCompletion { late_merge_key: Option, } -#[derive(serde::Serialize, serde::Deserialize)] struct OpenLlm { span_id: String, parent_span_id: String, @@ -141,14 +139,12 @@ struct OpenLlm { last_ms: i64, } -#[derive(serde::Serialize, serde::Deserialize)] struct OpenTool { span_id: String, parent_span_id: String, start_ms: i64, } -#[derive(serde::Serialize, serde::Deserialize)] struct CompletedTool { span_id: String, parent_span_id: String, @@ -158,7 +154,7 @@ struct CompletedTool { failure_recorded: bool, } -#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Default)] struct TranscriptCursor { path: Option, offset: u64, @@ -196,12 +192,11 @@ struct TranscriptBatch { reset: bool, } -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone)] struct PendingWork { event: Envelope, } -#[derive(serde::Serialize, serde::Deserialize)] struct GrokTranslator { session_id: String, session_span_id: String, @@ -226,7 +221,6 @@ struct GrokTranslator { first_llm_user_input: Option, pending: Option, cwd: Option, - #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, last_ts_ms: i64, } @@ -1228,7 +1222,6 @@ impl GrokTranslator { } impl AgentTranslator for GrokTranslator { - translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), diff --git a/bt-daemon/src/translate/mod.rs b/bt-daemon/src/translate/mod.rs index 90c707db..b4873397 100644 --- a/bt-daemon/src/translate/mod.rs +++ b/bt-daemon/src/translate/mod.rs @@ -7,25 +7,10 @@ //! whole pipeline be exercised with a debug sink and makes translators unit- //! testable without any network. -// Checkpoint codecs stay local to each concrete reducer; the producer owns -// format compatibility, compression, and the atomic SQLite input boundary. -macro_rules! translator_checkpoint { - () => { - fn save_state(&self) -> anyhow::Result>> { - Ok(Some(serde_json::to_vec(self)?)) - } - fn restore_state(&mut self, state: &[u8]) -> anyhow::Result<()> { - *self = serde_json::from_slice(state)?; - Ok(()) - } - }; -} - mod shared; pub use shared::CowSpanOp; mod antigravity; -mod checkpoint; mod claude; mod codex; pub(crate) mod cursor; @@ -162,15 +147,6 @@ pub struct SessionCtx { /// A per-session state machine. One instance per session; `&mut self` so it /// can hold open-span maps, transcript offsets, etc. pub trait AgentTranslator: Send { - /// Reducer context after a completely drained source event. These bytes - /// are versioned by the producer and committed alongside its input head. - fn save_state(&self) -> anyhow::Result>> { - Ok(None) - } - fn restore_state(&mut self, _state: &[u8]) -> anyhow::Result<()> { - anyhow::bail!("translator does not support checkpoints") - } - /// Handle one event, returning span ops to emit. fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result>; diff --git a/bt-daemon/src/translate/muse.rs b/bt-daemon/src/translate/muse.rs index 7fddd4e9..ff757aee 100644 --- a/bt-daemon/src/translate/muse.rs +++ b/bt-daemon/src/translate/muse.rs @@ -26,13 +26,12 @@ impl TranslatorFactory for crate::agents::Muse { Box::new(MuseTranslator::new(session_id, GitMetadataCache::shared())) } } -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone)] struct OpenOp { id: String, parents: Vec, turn_span_id: Option, } -#[derive(serde::Serialize, serde::Deserialize)] struct MuseTranslator { session_id: String, root_id: String, @@ -50,7 +49,6 @@ struct MuseTranslator { usage: UsageTracker, last_completed_turn: Option, last_ts: i64, - #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, } impl MuseTranslator { @@ -331,7 +329,6 @@ impl MuseTranslator { } } impl AgentTranslator for MuseTranslator { - translator_checkpoint!(); fn handle(&mut self, e: &Envelope, ctx: &SessionCtx) -> Result> { self.last_ts = self.last_ts.max(e.ts_ms); let event = match MuseEvent::decode(e) { diff --git a/bt-daemon/src/translate/muse_events.rs b/bt-daemon/src/translate/muse_events.rs index 93a46306..2e3de370 100644 --- a/bt-daemon/src/translate/muse_events.rs +++ b/bt-daemon/src/translate/muse_events.rs @@ -1,6 +1,6 @@ //! Typed tool, observation, subagent, and compaction reducers. use super::*; -#[derive(Default, serde::Serialize, serde::Deserialize)] +#[derive(Default)] pub(super) struct Events { open: HashMap, closed: RecentSet, diff --git a/bt-daemon/src/translate/muse_usage.rs b/bt-daemon/src/translate/muse_usage.rs index 98b20044..4a889322 100644 --- a/bt-daemon/src/translate/muse_usage.rs +++ b/bt-daemon/src/translate/muse_usage.rs @@ -3,7 +3,7 @@ use super::super::recent::RecentMap; use crate::muse::usage::{TokenMetrics, TokenUsage, UsageScope}; -#[derive(Default, serde::Serialize, serde::Deserialize)] +#[derive(Default)] pub(super) struct UsageTracker { turn_usage: RecentMap, model_metrics: RecentMap, diff --git a/bt-daemon/src/translate/opencode.rs b/bt-daemon/src/translate/opencode.rs index 079baab0..c487b0cf 100644 --- a/bt-daemon/src/translate/opencode.rs +++ b/bt-daemon/src/translate/opencode.rs @@ -340,7 +340,7 @@ fn properties(payload: &Value, field: &str) -> Option { .and_then(decode) } -#[derive(Default, serde::Serialize, serde::Deserialize)] +#[derive(Default)] struct NativeSession { root_span_id: String, effective_root_span_id: String, @@ -374,7 +374,6 @@ struct NativeSession { history_tool_results: HashMap>, } -#[derive(serde::Serialize, serde::Deserialize)] struct ClosedToolSpan { span_id: String, root_span_id: String, @@ -390,19 +389,17 @@ impl NativeSession { } } -#[derive(Default, serde::Serialize, serde::Deserialize)] +#[derive(Default)] struct MessageHistory { entries: Vec, pending_compaction: Option, } -#[derive(serde::Serialize, serde::Deserialize)] struct HistoryEntry { message_id: String, values: Vec, } -#[derive(serde::Serialize, serde::Deserialize)] struct CompactionBoundary { tail_start_id: Option, } @@ -496,7 +493,7 @@ impl MessageHistory { } } -#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Default)] struct PermissionRequest { id: Option, session_id: Option, @@ -507,18 +504,15 @@ struct PermissionRequest { permission_type: Option, } -#[derive(serde::Serialize, serde::Deserialize)] struct OpenCodeTranslator { daemon_session_id: String, sessions: HashMap, permission_requests: HashMap, - #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, last_ts_ms: i64, } impl AgentTranslator for OpenCodeTranslator { - translator_checkpoint!(); fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); let mut ops = match event.event.as_str() { diff --git a/bt-daemon/src/translate/pi.rs b/bt-daemon/src/translate/pi.rs index 93e90746..477df5a0 100644 --- a/bt-daemon/src/translate/pi.rs +++ b/bt-daemon/src/translate/pi.rs @@ -61,7 +61,7 @@ struct BeforeAgentStart { /// `ctx.model`: the selected model, which may be a Pi 1.0 virtual model that /// routes each request to a physical model. -#[derive(Deserialize, serde::Serialize)] +#[derive(Deserialize)] struct SelectedModel { provider: String, id: String, @@ -309,7 +309,6 @@ fn decode(value: &Value) -> Option { serde_json::from_value(value.clone()).ok() } -#[derive(serde::Serialize, serde::Deserialize)] struct PendingLlm { start_ms: i64, input: Value, @@ -317,14 +316,13 @@ struct PendingLlm { request_config: Map, selected_model: Option, } -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone)] struct ToolStart { start_ms: i64, name: String, args: Value, parent_call: Option, } -#[derive(serde::Serialize, serde::Deserialize)] struct PiTranslator { session_id: String, root_span_id: String, @@ -351,12 +349,10 @@ struct PiTranslator { branch_summary: Option<(String, i64, Value)>, last_ts: i64, thinking_level: Option, - #[serde(skip, default = "GitMetadataCache::shared")] git: Arc, } impl AgentTranslator for PiTranslator { - translator_checkpoint!(); fn handle(&mut self, envelope: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts = self.last_ts.max(envelope.ts_ms); let event = envelope.payload.get("event").unwrap_or(&envelope.payload); diff --git a/bt-daemon/src/translate/recent.rs b/bt-daemon/src/translate/recent.rs index 4b1412ec..557280be 100644 --- a/bt-daemon/src/translate/recent.rs +++ b/bt-daemon/src/translate/recent.rs @@ -11,8 +11,6 @@ use std::hash::Hash; pub(super) const RECENT_ID_CAPACITY: usize = 1_024; -#[derive(serde::Serialize, serde::Deserialize)] -#[serde(bound(deserialize = "K: Eq + Hash + serde::Deserialize<'de>"))] pub(super) struct RecentSet { values: HashSet, order: VecDeque, @@ -61,8 +59,6 @@ impl RecentSet { } } -#[derive(serde::Serialize, serde::Deserialize)] -#[serde(bound(deserialize = "K: Eq + Hash + serde::Deserialize<'de>, V: serde::Deserialize<'de>"))] pub(super) struct RecentMap { values: HashMap, order: VecDeque, diff --git a/bt-daemon/src/translate/tool.rs b/bt-daemon/src/translate/tool.rs index 49573345..aabe0ae6 100644 --- a/bt-daemon/src/translate/tool.rs +++ b/bt-daemon/src/translate/tool.rs @@ -6,7 +6,7 @@ use serde_json::{json, Map, Value}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ToolApproval { Approved, Denied, diff --git a/bt-daemon/src/translate/turn_lineage.rs b/bt-daemon/src/translate/turn_lineage.rs index 4006d227..50d466b3 100644 --- a/bt-daemon/src/translate/turn_lineage.rs +++ b/bt-daemon/src/translate/turn_lineage.rs @@ -45,13 +45,6 @@ fn publish_turn(row: &mut SpanRow) { } impl AgentTranslator for TurnLineage { - fn save_state(&self) -> anyhow::Result>> { - self.inner.save_state() - } - fn restore_state(&mut self, state: &[u8]) -> anyhow::Result<()> { - self.inner.restore_state(state) - } - fn handle( &mut self, event: &crate::wire::Envelope, From 653b55e9a46af14ac116edf2f712d88648da2ba6 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 3 Nov 2026 04:16:44 +0800 Subject: [PATCH 18/19] Preserve existing journal writes and Pi delta encoding --- bt-daemon/src/journal.rs | 536 ++++++++++++++------------ bt-daemon/src/pipeline/input.rs | 17 +- bt-daemon/src/pipeline/translation.rs | 22 +- bt-daemon/src/server/ingress.rs | 18 +- bt-daemon/src/server/mod.rs | 4 +- bt-daemon/src/server/sessions.rs | 6 +- bt-daemon/src/translate/mod.rs | 1 + bt-daemon/tests/pipeline.rs | 24 +- bt-daemon/tests/root_provenance.rs | 15 +- 9 files changed, 328 insertions(+), 315 deletions(-) diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index 8e0d1a29..809a3316 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -10,14 +10,11 @@ //! still tell which delivery pipelines a managed child produced after the //! daemon that accepted them has restarted or idle-exited. -use crate::pipeline::input::SourcePosition; -pub use crate::pipeline::input::{ControlKind, PipelineControl}; - use crate::wire::{BackendAuth, Envelope, RedactedEnvelope, SessionRoute}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::path::{Path, PathBuf}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; pub fn journal_dir(data_dir: &Path) -> PathBuf { data_dir.join("journal") @@ -224,33 +221,7 @@ pub async fn gc_old_managed_runs(data_dir: &Path, max_age: std::time::Duration) pub struct JournalWriter { file: tokio::fs::File, position: u64, - failed_write: bool, -} - -/// Ignore only an unterminated trailing write. Complete records, including -/// malformed ones, remain visible so corruption cannot silently discard data. -async fn complete_position(file: &mut tokio::fs::File) -> std::io::Result { - let len = file.metadata().await?.len(); - if len == 0 { - return Ok(0); - } - file.seek(std::io::SeekFrom::Start(len - 1)).await?; - if file.read_u8().await? == b'\n' { - return Ok(len); - } - let mut end = len; - let mut buffer = vec![0; 8192]; - while end > 0 { - let start = end.saturating_sub(buffer.len() as u64); - let bytes = &mut buffer[..(end - start) as usize]; - file.seek(std::io::SeekFrom::Start(start)).await?; - file.read_exact(bytes).await?; - if let Some(index) = bytes.iter().rposition(|byte| *byte == b'\n') { - return Ok(start + index as u64 + 1); - } - end = start; - } - Ok(0) + pi_context: Vec, } impl JournalWriter { @@ -258,21 +229,16 @@ impl JournalWriter { if let Some(dir) = path.parent() { tokio::fs::create_dir_all(dir).await?; } - let mut file = tokio::fs::OpenOptions::new() + let file = tokio::fs::OpenOptions::new() .create(true) - .read(true) .append(true) .open(path) .await?; - let position = complete_position(&mut file).await?; - if position < file.metadata().await?.len() { - file.set_len(position).await?; - } - file.sync_data().await?; + let position = file.metadata().await?.len(); Ok(Self { file, position, - failed_write: false, + pi_context: Vec::new(), }) } @@ -280,91 +246,38 @@ impl JournalWriter { self.position } - async fn append_bytes(&mut self, line: &[u8]) -> anyhow::Result { - if self.failed_write { - self.file.set_len(self.position).await?; - self.file.sync_data().await?; - self.failed_write = false; - } - if let Err(error) = async { - self.file.write_all(line).await?; - self.file.flush().await?; - self.file.sync_data().await - } - .await - { - self.failed_write = true; - return Err(error.into()); - } - let before = self.position; - self.position += line.len() as u64; - Ok(before) - } - /// Append one redacted event and confirm its durable write before capture acknowledges it. /// - /// Complete journal records are never capped or truncated. An unfinished - /// trailing write is discarded on reopen, before new appends. Its size is - /// managed by age-based GC. Native event payloads are preserved, including - /// repeated message snapshots. External transcript mirrors capture each - /// byte once, and replay streams records without loading the whole log. - #[cfg(test)] + /// The journal is a durability record and is never capped or truncated: + /// dropping entries would silently cost recovery fidelity. Its size is + /// bounded instead by writing each transcript byte once (see + /// [`crate::transcript_mirror`]) and by age-based GC, and replay reads it + /// as a stream so a large journal never becomes a large allocation. pub async fn append(&mut self, env: &Envelope) -> anyhow::Result { - self.append_ordered(env, None) - .await - .map(|position| position.through) - } - - pub(crate) async fn append_ordered( - &mut self, - env: &Envelope, - sequence: Option, - ) -> anyhow::Result { - #[derive(Serialize)] - struct Captured<'a> { - #[serde(flatten)] - envelope: &'a RedactedEnvelope, - _bt_record_type: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - _bt_capture_sequence: Option, - } - let redacted = env.redacted(); - let mut line = serde_json::to_vec(&Captured { - envelope: &redacted, - _bt_record_type: "event", - _bt_capture_sequence: sequence, - })?; + let mut redacted = env.redacted(); + compact_pi_payload(&mut redacted, &mut self.pi_context); + let mut line = serde_json::to_vec(&redacted)?; line.push(b'\n'); - let before = self.append_bytes(&line).await?; - Ok(SourcePosition { - before, - through: self.position, - }) + self.file.write_all(&line).await?; + self.file.flush().await?; + self.file.sync_data().await?; + self.position += line.len() as u64; + Ok(self.position) } - #[cfg(test)] pub(crate) async fn append_control( &mut self, control: &PipelineControl, ) -> anyhow::Result { - self.append_control_record(control) - .await - .map(|position| position.through) - } - - pub(crate) async fn append_control_record( - &mut self, - control: &PipelineControl, - ) -> anyhow::Result { let mut value = serde_json::to_value(control)?; value["_bt_record_type"] = "pipeline_control".into(); let mut line = serde_json::to_vec(&value)?; line.push(b'\n'); - let before = self.append_bytes(&line).await?; - Ok(SourcePosition { - before, - through: self.position, - }) + self.file.write_all(&line).await?; + self.file.flush().await?; + self.file.sync_data().await?; + self.position += line.len() as u64; + Ok(self.position) } } @@ -386,13 +299,25 @@ pub enum JournalRecord { DeliveryCheckpoint { route: SessionRoute, through: u64 }, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ControlKind { + Checkpoint, + Finalize, + Retire, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PipelineControl { + pub kind: ControlKind, + pub bounds: std::collections::HashMap, +} + #[derive(Debug)] pub struct JournalRecordEntry { pub record: JournalRecord, /// Exclusive byte offset after this record in the journal. pub through: u64, - /// Daemon-local capture order. Kept outside the native event payload. - pub(crate) capture_sequence: Option, } /// Streaming reader over one session's journal. @@ -404,7 +329,7 @@ pub struct JournalReader { path: PathBuf, line_no: usize, position: u64, - legacy_pi: Vec, + pi_context: Vec, } impl JournalReader { @@ -412,30 +337,22 @@ impl JournalReader { self.position } - /// Seek within new, self-contained records. Legacy delta streams must be - /// read from the beginning to reconstruct their prior message context. + /// Rebuild decoder context before continuing at a saved record boundary. pub(crate) async fn resume(path: &Path, position: u64, through: u64) -> anyhow::Result { - let mut file = tokio::fs::File::open(path).await?; - anyhow::ensure!( - position <= through && through <= file.metadata().await?.len(), - "checkpoint is beyond the source journal" - ); - if position > 0 { - file.seek(std::io::SeekFrom::Start(position - 1)).await?; + anyhow::ensure!(position <= through, "invalid recovery range"); + let mut reader = Self::open(path, through) + .await? + .ok_or_else(|| anyhow::anyhow!("recovery journal is missing"))?; + while reader.position < position { anyhow::ensure!( - file.read_u8().await? == b'\n', - "checkpoint is not at a journal record boundary" + reader.next_record().await?.is_some(), + "recovery journal ended before its boundary" ); } - file.seek(std::io::SeekFrom::Start(position)).await?; - Ok(Self { - reader: tokio::io::BufReader::new(file.take(through - position)), - path: path.to_owned(), - line_no: 0, - position, - legacy_pi: Vec::new(), - }) + anyhow::ensure!(reader.position == position, "invalid recovery boundary"); + Ok(reader) } + /// Open a journal for streaming, reading at most `through` bytes. /// /// That bound is what keeps replay from consuming the very event that @@ -454,17 +371,17 @@ impl JournalReader { path: path.to_path_buf(), line_no: 0, position: 0, - legacy_pi: Vec::new(), + pi_context: Vec::new(), })) } /// The journal's current length, which is the bound a session created now /// should replay through. A missing journal replays nothing. pub async fn recorded_len(path: &Path) -> u64 { - let Ok(mut file) = tokio::fs::File::open(path).await else { - return 0; - }; - complete_position(&mut file).await.unwrap_or(0) + tokio::fs::metadata(path) + .await + .map(|meta| meta.len()) + .unwrap_or(0) } /// The next entry, or `None` at end of file. @@ -488,24 +405,18 @@ impl JournalReader { } self.position += read as u64; self.line_no += 1; - anyhow::ensure!( - bytes.last() == Some(&b'\n'), - "unterminated source journal record" - ); let path = self.path.clone(); let line_no = self.line_no; let position = self.position; - let mut decoder = std::mem::take(&mut self.legacy_pi); - // Expanding and parsing a large history can be CPU-heavy even - // when file reads are async. Never run it on the hook IPC thread. - let (decoder, decoded) = tokio::task::spawn_blocking(move || { - let decoded = decode_record(&bytes, &path, line_no, position, &mut decoder); - (decoder, decoded) + let mut pi_context = std::mem::take(&mut self.pi_context); + let (pi_context, record) = tokio::task::spawn_blocking(move || { + let record = decode_record(&bytes, &path, line_no, position, &mut pi_context); + (pi_context, record) }) .await?; - self.legacy_pi = decoder; - if let Some(entry) = decoded? { - return Ok(Some(entry)); + self.pi_context = pi_context; + if let Some(record) = record? { + return Ok(Some(record)); } } } @@ -521,9 +432,6 @@ impl JournalReader { /// journal being recovered. Older journals contain no checkpoints and /// therefore retain their existing replay behavior. pub async fn acknowledged_through(path: &Path, through: u64, route: &SessionRoute) -> u64 { - if through == 0 { - return 0; - } let Ok(Some(mut reader)) = Self::open(path, through).await else { return 0; }; @@ -548,7 +456,7 @@ fn decode_record( path: &Path, line_no: usize, position: u64, - legacy_pi: &mut Vec, + pi_context: &mut Vec, ) -> anyhow::Result> { let line = std::str::from_utf8(bytes) .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))? @@ -558,9 +466,6 @@ fn decode_record( } let value: serde_json::Value = serde_json::from_str(line) .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))?; - let capture_sequence = value - .get("_bt_capture_sequence") - .and_then(serde_json::Value::as_u64); let record = if value .get("_bt_record_type") .and_then(serde_json::Value::as_str) @@ -579,33 +484,83 @@ fn decode_record( { JournalRecord::Control(serde_json::from_value(value)?) } else { - let record_type = value - .get("_bt_record_type") - .and_then(serde_json::Value::as_str) - .map(str::to_owned); - let mut event: RedactedEnvelope = serde_json::from_value(value) + let mut event = serde_json::from_value(value) + .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))?; + expand_pi_payload(&mut event, pi_context) .map_err(|error| anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no))?; - match record_type.as_deref() { - Some("event") => legacy_pi.clear(), - None => expand_legacy_pi_payload(&mut event, legacy_pi).map_err(|error| { - anyhow::anyhow!("journal {}:{}: {error}", path.display(), line_no) - })?, - Some(kind) => anyhow::bail!("unsupported journal record type {kind:?}"), - } JournalRecord::Event(event) }; Ok(Some(JournalRecordEntry { record, through: position, - capture_sequence, })) } -const LEGACY_PI_MESSAGES_DELTA: &str = "_bt_messages_delta"; +const PI_MESSAGES_DELTA: &str = "_bt_messages_delta"; + +fn compact_pi_payload(event: &mut RedactedEnvelope, previous: &mut Vec) { + if event.source != "pi" { + return; + } + let Some(native) = event + .payload + .get_mut("event") + .and_then(serde_json::Value::as_object_mut) + else { + return; + }; + match event.event.as_str() { + "context" => { + let Some(messages) = native + .remove("messages") + .and_then(|messages| messages.as_array().cloned()) + else { + return; + }; + let common_prefix = previous + .iter() + .zip(&messages) + .take_while(|(left, right)| left == right) + .count(); + native.insert( + PI_MESSAGES_DELTA.into(), + serde_json::json!({ + "common_prefix": common_prefix, + "suffix": messages[common_prefix..], + }), + ); + *previous = messages; + } + "before_provider_request" => { + // The translator records only the allowlisted request + // configuration and tool definitions, never the prompt or messages. + if let Some(payload) = native.get_mut("payload") { + *payload = serde_json::Value::Object(crate::translate::pi_request_config(payload)); + } + } + "agent_end" => { + // The translator only consumes willRetry from this lifecycle event. + native.remove("messages"); + } + "message_update" => { + // Pi repeats the progressively growing assistant message on every + // streaming update. Translation only needs the update type to mark + // time-to-first-token; the completed message arrives separately in + // message_end. + native.remove("message"); + if let Some(update) = native + .get_mut("assistantMessageEvent") + .and_then(serde_json::Value::as_object_mut) + { + update.retain(|key, _| key == "type"); + } + native.retain(|key, _| key == "type" || key == "assistantMessageEvent"); + } + _ => {} + } +} -// Read compatibility for journals written before native payload preservation. -// New event records are tagged explicitly and never enter this decoder. -fn expand_legacy_pi_payload( +fn expand_pi_payload( event: &mut RedactedEnvelope, previous: &mut Vec, ) -> anyhow::Result<()> { @@ -619,7 +574,7 @@ fn expand_legacy_pi_payload( else { return Ok(()); }; - let Some(delta) = native.remove(LEGACY_PI_MESSAGES_DELTA) else { + let Some(delta) = native.remove(PI_MESSAGES_DELTA) else { if let Some(messages) = native.get("messages").and_then(serde_json::Value::as_array) { *previous = messages.clone(); } @@ -767,6 +722,7 @@ pub fn envelope_from_redacted(r: RedactedEnvelope) -> Envelope { #[cfg(test)] mod tests { use super::*; + fn pi_context(messages: Vec, ts_ms: i64) -> Envelope { Envelope { source: "pi".into(), @@ -784,76 +740,184 @@ mod tests { } #[tokio::test] - async fn released_pi_deltas_remain_readable_alongside_complete_new_records() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("legacy.ndjson"); - let first = pi_context(vec![serde_json::json!({"role":"user","content":"old"})], 1); - let mut delta = pi_context(Vec::new(), 2); - delta.payload = serde_json::json!({"event":{"_bt_messages_delta":{"common_prefix":1,"suffix":[{"role":"assistant","content":"answer"}]}}}); - tokio::fs::write( - &path, - format!( - "{}\n{}\n", - serde_json::to_string(&first.redacted()).unwrap(), - serde_json::to_string(&delta.redacted()).unwrap() - ), - ) - .await - .unwrap(); + async fn pi_context_history_is_delta_encoded_and_replayed() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("pi.ndjson"); let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let mut native = first.clone(); - native.payload["event"]["_bt_messages_delta"] = serde_json::json!({"native":"preserve"}); - let through = writer.append(&native).await.unwrap(); - let mut reader = JournalReader::open(&path, through).await.unwrap().unwrap(); - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - first.payload + let mut messages = Vec::new(); + let mut naive_bytes = 0usize; + for index in 0..20 { + messages.push(serde_json::json!({ + "role":"user", + "content": format!("{index}:{}", "x".repeat(4096)), + })); + let event = pi_context(messages.clone(), index); + naive_bytes += serde_json::to_vec(&event.redacted()).unwrap().len() + 1; + writer.append(&event).await.unwrap(); + } + let compacted = vec![ + serde_json::json!({"role":"compactionSummary","summary":"bounded"}), + serde_json::json!({"role":"user","content":"after"}), + ]; + let event = pi_context(compacted.clone(), 20); + naive_bytes += serde_json::to_vec(&event.redacted()).unwrap().len() + 1; + writer.append(&event).await.unwrap(); + drop(writer); + + let stored = tokio::fs::read(&path).await.unwrap(); + assert!( + stored.len() < naive_bytes / 3, + "Pi journal did not eliminate cumulative context copies: stored={} naive={naive_bytes}", + stored.len() ); - let expanded = reader.next_entry().await.unwrap().unwrap(); + assert!(!String::from_utf8_lossy(&stored).contains("\"messages\"")); + + let through = stored.len() as u64; + let mut reader = JournalReader::open(&path, through).await.unwrap().unwrap(); + let mut contexts = Vec::new(); + while let Some(entry) = reader.next_entry().await.unwrap() { + contexts.push( + entry + .payload + .pointer("/event/messages") + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap(), + ); + } + assert_eq!(contexts.len(), 21); + assert_eq!(contexts[19].len(), 20); + assert_eq!(contexts[20], compacted); + } + + #[tokio::test] + async fn pi_streaming_updates_do_not_persist_growing_partial_messages() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("pi.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let marker = "growing-partial-marker".repeat(4096); + let event = Envelope { + source: "pi".into(), + source_version: None, + plugin_version: None, + session_id: "pi-session".into(), + event: "message_update".into(), + ts_ms: 1, + managed_run_id: None, + capture: None, + payload: serde_json::json!({ + "event": { + "type": "message_update", + "assistantMessageEvent": { + "type": "text_delta", + "partial": marker, + }, + "message": {"role": "assistant", "content": marker}, + } + }), + route: None, + config: None, + }; + writer.append(&event).await.unwrap(); + drop(writer); + + let stored = tokio::fs::read(&path).await.unwrap(); + assert!(!String::from_utf8_lossy(&stored).contains("growing-partial-marker")); + + let mut reader = JournalReader::open(&path, stored.len() as u64) + .await + .unwrap() + .unwrap(); + let replayed = reader.next_entry().await.unwrap().unwrap(); assert_eq!( - expanded.payload["event"]["messages"], - serde_json::json!([{"role":"user","content":"old"},{"role":"assistant","content":"answer"}]) + replayed + .payload + .pointer("/event/assistantMessageEvent/type"), + Some(&serde_json::json!("text_delta")) ); + } + + #[tokio::test] + async fn pi_provider_requests_persist_only_request_configuration() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("pi.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + let event = Envelope { + source: "pi".into(), + source_version: None, + plugin_version: None, + session_id: "pi-session".into(), + event: "before_provider_request".into(), + ts_ms: 1, + managed_run_id: None, + capture: None, + payload: serde_json::json!({ + "event": { + "type": "before_provider_request", + "payload": { + "system": "system-prompt-marker", + "messages": [{"role": "user", "content": "message-marker"}], + "tools": [{"name": "read", "input_schema": {"type": "object"}}], + "max_tokens": 1024, + }, + } + }), + route: None, + config: None, + }; + writer.append(&event).await.unwrap(); + drop(writer); + + let stored = tokio::fs::read(&path).await.unwrap(); + let stored_text = String::from_utf8_lossy(&stored); + assert!(!stored_text.contains("system-prompt-marker")); + assert!(!stored_text.contains("message-marker")); + + let mut reader = JournalReader::open(&path, stored.len() as u64) + .await + .unwrap() + .unwrap(); + let replayed = reader.next_entry().await.unwrap().unwrap(); assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - native.payload + replayed.payload.pointer("/event/payload"), + Some(&serde_json::json!({ + "max_tokens": 1024, + "tools": [{"type": "function", "function": { + "name": "read", + "parameters": {"type": "object"}, + }}], + })) ); } - #[tokio::test] - async fn partial_writes_recover_on_retry_and_reopen_without_losing_payloads() { - for reopen in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("events.ndjson"); - let mut writer = JournalWriter::open_path(&path).await.unwrap(); - let first = pi_context( - vec![serde_json::json!({"content":"history".repeat(1024)})], - 1, + #[test] + fn pi_journaled_request_configuration_replays_unchanged() { + let schema = serde_json::json!({"type": "object"}); + for payload in [ + serde_json::json!({ + "max_tokens": 1024, + "tools": [ + {"name": "read", "input_schema": schema, "strict": true}, + {"type": "web_search_20250305", "name": "web_search"}, + ], + "tool_choice": {"type": "tool", "name": "read", "disable_parallel_tool_use": true}, + }), + serde_json::json!({"config": { + "temperature": 0.5, + "tools": [{"functionDeclarations": [{"name": "read", "parameters": schema}]}, {"googleSearch": {}}], + "toolConfig": {"functionCallingConfig": {"mode": "ANY"}}, + }}), + serde_json::json!({ + "toolConfig": {"tools": [{"toolSpec": {"name": "read", "inputSchema": {"json": schema}}}]}, + "inferenceConfig": {"maxTokens": 512}, + }), + ] { + let journaled = + serde_json::Value::Object(crate::translate::pi_request_config(&payload)); + assert_eq!( + serde_json::Value::Object(crate::translate::pi_request_config(&journaled)), + journaled ); - let through = writer.append(&first).await.unwrap(); - writer.file.write_all(b"{unfinished").await.unwrap(); - writer.file.flush().await.unwrap(); - assert_eq!(JournalReader::recorded_len(&path).await, through); - if reopen { - writer = JournalWriter::open_path(&path).await.unwrap(); - } else { - writer.failed_write = true; - } - let mut second = first.clone(); - second.payload["tail"] = "after recovery".into(); - let position = writer.append_ordered(&second, None).await.unwrap(); - assert_eq!(position.before, through); - let mut reader = JournalReader::open(&path, position.through) - .await - .unwrap() - .unwrap(); - for event in [first, second] { - assert_eq!( - reader.next_entry().await.unwrap().unwrap().payload, - event.payload - ); - } - assert!(reader.next_entry().await.unwrap().is_none()); } } diff --git a/bt-daemon/src/pipeline/input.rs b/bt-daemon/src/pipeline/input.rs index 5aa1b985..63d5586c 100644 --- a/bt-daemon/src/pipeline/input.rs +++ b/bt-daemon/src/pipeline/input.rs @@ -1,9 +1,8 @@ //! Stage input shared by direct capture and recovery. Storage supplies only //! byte positions; original live payloads arrive from capture. +pub use crate::journal::PipelineControl; use crate::wire::Envelope; use parking_lot::Mutex; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::{collections::VecDeque, sync::Arc}; const RECORD_BUDGET: usize = 64; @@ -33,20 +32,6 @@ impl SourceRecord { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ControlKind { - Checkpoint, - Finalize, - Retire, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PipelineControl { - pub kind: ControlKind, - pub bounds: HashMap, -} - #[derive(Default)] struct Buffer { records: VecDeque, diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 3716c3fa..1755bf8e 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -492,8 +492,9 @@ impl Producer { if writer.position() > self.position { return Ok(None); } - let position = writer.append_control_record(&control).await?; - let through = position.through; + let before = writer.position(); + let through = writer.append_control(&control).await?; + let position = SourcePosition { before, through }; self.options.input.push(SourceRecord { event: SourceEvent::Control(control), position, @@ -884,8 +885,9 @@ pub(crate) mod tests { } pub(crate) async fn write(&self, event: &crate::wire::Envelope, live: bool) -> u64 { let mut writer = self.options.events.lock().await; - let position = writer.append_ordered(event, None).await.unwrap(); - let through = position.through; + let before = writer.position(); + let through = writer.append(event).await.unwrap(); + let position = SourcePosition { before, through }; if live { self.options.input.push_event(event.clone(), position); } @@ -934,7 +936,7 @@ pub(crate) mod tests { covered.push(agent.clone()); events.insert(0, events[0].clone()); for event in &mut events { - event.payload["codec_padding"] = serde_json::json!("é📚history".repeat(1024)); + event.payload["padding"] = serde_json::json!("é📚history".repeat(1024)); } let mut cases = vec![("live", 0), ("overflow", 2)]; cases.extend((1..=events.len()).map(|split| ("cold", split))); @@ -946,16 +948,6 @@ pub(crate) mod tests { positions.push(f.write(event, mode == "live" || mode == "overflow").await); } let tail = *positions.last().unwrap(); - if mode == "cold" && split == 1 { - let stored = tokio::fs::read_to_string(&f.options.event_path) - .await - .unwrap(); - assert_eq!(stored.lines().count(), events.len()); - for (line, event) in stored.lines().zip(&events) { - let record: serde_json::Value = serde_json::from_str(line).unwrap(); - assert_eq!(record["payload"], event.payload); - } - } match mode { "live" => { f.process(positions[1]).await; diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 55e16fe8..9c2f4abb 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -69,8 +69,8 @@ pub(super) async fn drain_capture_buffer(daemon: &Arc) -> anyhow::Result } } -/// Read frozen evicted ranges, merging one lookahead per source in capture -/// order. Current captures contain complete payloads and need no prefix replay. +/// Read frozen evicted ranges in source order, merging by event timestamp as +/// startup recovery does. Rebuild decoder context before each range. pub(super) async fn replay_evicted( daemon: &Arc, bounds: &HashMap, @@ -78,10 +78,10 @@ pub(super) async fn replay_evicted( let mut ready = Vec::new(); for (path, range) in bounds { let mut reader = journal::JournalReader::resume(path, range.from, range.through).await?; - if let Some((sequence, event)) = + if let Some((timestamp, event)) = next_evicted_event(daemon, &mut reader, range.through).await? { - ready.push((sequence, event, reader, range.through)); + ready.push((timestamp, event, reader, range.through)); } } while let Some(index) = ready @@ -92,8 +92,8 @@ pub(super) async fn replay_evicted( { let (_, event, mut reader, through) = ready.swap_remove(index); dispatch_ingress_event(daemon, event).await; - if let Some((sequence, event)) = next_evicted_event(daemon, &mut reader, through).await? { - ready.push((sequence, event, reader, through)); + if let Some((timestamp, event)) = next_evicted_event(daemon, &mut reader, through).await? { + ready.push((timestamp, event, reader, through)); } } Ok(()) @@ -103,7 +103,7 @@ async fn next_evicted_event( daemon: &Daemon, reader: &mut journal::JournalReader, through: u64, -) -> anyhow::Result> { +) -> anyhow::Result> { while reader.position() < through { let before = reader.position(); let entry = reader @@ -118,9 +118,7 @@ async fn next_evicted_event( }; env.source = source.to_owned(); return Ok(Some(( - entry - .capture_sequence - .ok_or_else(|| anyhow::anyhow!("captured event has no sequence"))?, + env.ts_ms, PendingEvent { env, replay_through: before, diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index ed9ca2d2..2b25a34f 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -45,7 +45,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; @@ -137,7 +137,6 @@ pub struct Daemon { /// Recovery must not absorb queued live events into its historical head. startup_cutoffs: Mutex>>, capture_buffer: Mutex, - capture_sequence: AtomicU64, started: Instant, last_activity: Mutex, /// Prevents a shutdown drain from racing capture that has passed the @@ -190,7 +189,6 @@ impl Daemon { ingress_tx, startup_cutoffs: Mutex::new(startup_cutoffs), capture_buffer: Mutex::new(CaptureBuffer::default()), - capture_sequence: AtomicU64::new(0), started: Instant::now(), last_activity: Mutex::new(Instant::now()), capture_gate: tokio::sync::RwLock::new(()), diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index 2c63f913..67e5000c 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -325,10 +325,8 @@ impl Daemon { let path = journal::source_journal_path(&self.data_dir, &env.source, &env.session_id); cutoffs.entry(path).or_insert(before); } - // Related hooks cannot begin until their predecessor's durable append - // is acknowledged. Concurrent sessions need no stronger total order. - let sequence = self.capture_sequence.fetch_add(1, Ordering::Relaxed); - writer.append_ordered(env, Some(sequence)).await + let through = writer.append(env).await?; + Ok(crate::pipeline::input::SourcePosition { before, through }) } pub(super) fn session_lock( diff --git a/bt-daemon/src/translate/mod.rs b/bt-daemon/src/translate/mod.rs index b4873397..40780482 100644 --- a/bt-daemon/src/translate/mod.rs +++ b/bt-daemon/src/translate/mod.rs @@ -22,6 +22,7 @@ mod muse; mod opencode; mod pi; pub(crate) use claude::successful_subagent_handback_message; +pub(crate) use pi::request_config as pi_request_config; mod origins; mod recent; pub(crate) use origins::WeakOrigins; diff --git a/bt-daemon/tests/pipeline.rs b/bt-daemon/tests/pipeline.rs index 0590902d..e6ee0146 100644 --- a/bt-daemon/tests/pipeline.rs +++ b/bt-daemon/tests/pipeline.rs @@ -1257,12 +1257,7 @@ async fn events_are_ordered_journaled_and_emitted() { let events: Vec = jlines .iter() - .filter(|line| { - serde_json::from_str::(line) - .unwrap() - .get("event") - .is_some() - }) + .filter(|line| !line.contains("\"_bt_record_type\"")) .map(|l| { serde_json::from_str::(l).unwrap()["event"] .as_str() @@ -1314,10 +1309,7 @@ async fn session_ids_longer_than_a_file_name_are_journaled_and_emitted() { assert_eq!( jtext .lines() - .filter(|line| serde_json::from_str::(line) - .unwrap() - .get("event") - .is_some()) + .filter(|line| !line.contains("\"_bt_record_type\"")) .count(), 2 ); @@ -1984,10 +1976,7 @@ async fn cold_worker_rebuilds_acknowledged_journal_without_redelivery() { assert_eq!( journal .lines() - .filter(|line| serde_json::from_str::(line) - .unwrap() - .get("event") - .is_some()) + .filter(|line| !line.contains("\"_bt_record_type\"")) .count(), 2, "the event history remains separate from ordered pipeline controls" @@ -2543,10 +2532,9 @@ async fn claude_journal_does_not_grow_with_the_transcript_on_every_turn() { // Re-journaling the transcript every turn would put this near // TURNS * transcript_len / 2 (tens of megabytes). References are ~a few - // hundred bytes each. Allow for longer Windows paths and concurrent - // transcript observations while remaining far below the duplicated data. + // hundred bytes each. assert!( - journal_len < 256 * 1024, + journal_len < 64 * 1024, "journal grew with the transcript: {journal_len} bytes for {TURNS} events \ over a {transcript_len}-byte transcript" ); @@ -3501,7 +3489,7 @@ async fn packaged_grok_hook_replays_bounded_transcripts_to_isolated_debug_routes let journal: Vec = journal_text .lines() .map(|line| serde_json::from_str(line).unwrap()) - .filter(|record: &serde_json::Value| record.get("event").is_some()) + .filter(|record: &serde_json::Value| record.get("_bt_record_type").is_none()) .collect(); assert_eq!(journal.len(), 2); assert_eq!(journal[0]["source"], "grok"); diff --git a/bt-daemon/tests/root_provenance.rs b/bt-daemon/tests/root_provenance.rs index 5ed19b84..3613c675 100644 --- a/bt-daemon/tests/root_provenance.rs +++ b/bt-daemon/tests/root_provenance.rs @@ -493,19 +493,8 @@ async fn legacy_acknowledged_spans_receive_late_updates_without_inventing_origin let raw = std::fs::read_to_string(&journal).unwrap(); let mut history = raw .lines() - .filter_map(|line| { - let mut record: Value = serde_json::from_str(line).unwrap(); - record.get("event")?; - // These small fixture payloads are full records. Remove the new - // framing to model the untagged legacy format faithfully. - assert_eq!(record["_bt_record_type"], "event"); - record.as_object_mut().unwrap().remove("_bt_record_type"); - record - .as_object_mut() - .unwrap() - .remove("_bt_capture_sequence"); - Some(format!("{record}\n")) - }) + .filter(|line| !line.contains("_bt_record_type")) + .map(|line| format!("{line}\n")) .collect::(); let through = history.len() as u64; history.push_str(&format!("{}\n",json!({"_bt_record_type":"delivery_checkpoint", "route":capture("UserPromptSubmit","v1","native1",1_000).route, "through":through}))); From 517c083c462f0c3b23bc1ffa96831390025ef37d Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 3 Nov 2026 17:37:08 +0800 Subject: [PATCH 19/19] Harden concurrent recovery and verify ordered trace equivalence --- bt-daemon/src/pipeline/journal.rs | 15 ++++++++++++++- bt-daemon/src/pipeline/translation.rs | 18 ++++++++++++++---- bt-daemon/src/server/ingress.rs | 21 +++++++++++++-------- bt-daemon/src/server/maintenance.rs | 13 +++++++++++++ 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/bt-daemon/src/pipeline/journal.rs b/bt-daemon/src/pipeline/journal.rs index a1140c98..c854ed26 100644 --- a/bt-daemon/src/pipeline/journal.rs +++ b/bt-daemon/src/pipeline/journal.rs @@ -184,7 +184,20 @@ impl SpanJournal { std::fs::create_dir_all(parent)?; let db = Connection::open(path)?; db.busy_timeout(std::time::Duration::from_secs(5))?; - db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; BEGIN IMMEDIATE;")?; + // Concurrent WAL transitions can return BUSY without invoking + // SQLite's busy handler. Retry initialization within the same + // bound, before any schema transaction has started. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + match db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; BEGIN IMMEDIATE;") { + Ok(()) => break, + Err(error) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DatabaseBusy) + && std::time::Instant::now() < deadline => { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + Err(error) => return Err(error.into()), + } + } // Read the version under the same write transaction as every // upgrade, so another opener cannot run a stale migration plan. let version: u32 = db.query_row("PRAGMA user_version", [], |row| row.get(0))?; diff --git a/bt-daemon/src/pipeline/translation.rs b/bt-daemon/src/pipeline/translation.rs index 1755bf8e..52bda1bb 100644 --- a/bt-daemon/src/pipeline/translation.rs +++ b/bt-daemon/src/pipeline/translation.rs @@ -938,14 +938,17 @@ pub(crate) mod tests { for event in &mut events { event.payload["padding"] = serde_json::json!("é📚history".repeat(1024)); } - let mut cases = vec![("live", 0), ("overflow", 2)]; + let mut cases = vec![("journal", 0), ("live", 0), ("overflow", 2)]; + cases.extend((1..events.len()).map(|split| ("gap", split))); cases.extend((1..=events.len()).map(|split| ("cold", split))); let mut expected = None; for (mode, split) in cases { let mut f = SourceFixture::new(&agent).await; let mut positions = vec![0]; - for event in &events { - positions.push(f.write(event, mode == "live" || mode == "overflow").await); + for (index, event) in events.iter().enumerate() { + let live = + mode == "live" || mode == "overflow" || (mode == "gap" && index != split); + positions.push(f.write(event, live).await); } let tail = *positions.last().unwrap(); match mode { @@ -964,6 +967,11 @@ pub(crate) mod tests { assert!(f.options.input.take(positions[1], positions[2]).is_some()); f.poison(positions[2]..tail).await; } + "gap" => { + // Rebuild decoder context after live input, then return + // to live input without rereading its retained tail. + f.poison(positions[split + 1]..tail).await; + } _ => { f.process(positions[split]).await; f.source.shutdown().await; @@ -976,7 +984,9 @@ pub(crate) mod tests { // StoredOp already includes the private correlation fields; // compare its complete encoding rather than duplicate its codec. let db = f.db(); - let mut query = db.prepare("SELECT value FROM ops ORDER BY value").unwrap(); + let mut query = db + .prepare("SELECT value FROM ops ORDER BY position") + .unwrap(); let rows: Vec> = query .query_map([], |row| row.get(0)) .unwrap() diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 9c2f4abb..c74c2eb9 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -63,7 +63,18 @@ pub(super) async fn drain_capture_buffer(daemon: &Arc) -> anyhow::Result loop { match daemon.next_capture_work() { CaptureWork::Evicted(bounds) => replay_evicted(daemon, &bounds).await?, - CaptureWork::Event(event) => dispatch_ingress_event(daemon, *event).await, + CaptureWork::Event(event) => { + daemon + .source_input_for(&event.env.source, &event.env.session_id) + .push_event( + event.env.clone(), + crate::pipeline::input::SourcePosition { + before: event.replay_through, + through: event.journal_through, + }, + ); + dispatch_ingress_event(daemon, *event).await; + } CaptureWork::Finished => return Ok(()), } } @@ -349,19 +360,13 @@ impl Daemon { .journal_writer_for(&env.source, &env.session_id) .await .map_err(|error| format!("journal failed: {error}"))?; - let input = self.source_input_for(&env.source, &env.session_id); let (replay_through, journal_through) = { let mut writer = writer.lock().await; - // Storage is a side branch: it receives a copy and returns only - // durable position/context metadata. The stage receives the - // original complete event directly from capture. let position = self .append_to_journal(&env, &mut writer) .await .map_err(|error| format!("journal failed: {error}"))?; - let bounds = (position.before, position.through); - input.push_event(env.clone(), position); - bounds + (position.before, position.through) }; let event = PendingEvent { env, diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index 3de48e0d..442b7df3 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -522,6 +522,10 @@ mod tests { .unwrap() .contains("next")); assert!(daemon.translations.lock().is_empty()); + assert!( + daemon.source_inputs.lock().is_empty(), + "capture must retain live payloads only in the bounded ingress queue" + ); let recovered = discover_unprocessed_journals(&daemon).await; assert_eq!(recovered.len(), 1); assert_eq!(recovered[0].journal_through, old_end); @@ -593,6 +597,15 @@ mod tests { ready.send(()).unwrap(); daemon.settle_ingress().await; for (slot, env) in envs.iter().enumerate() { + // These sources may have no authenticated delivery actor. Wait for + // translation itself before inspecting its durable head. + daemon + .translation_for(env) + .await + .unwrap() + .barrier() + .await + .unwrap(); let path = journal::source_journal_path(tmp.path(), &env.source, &env.session_id); let storage = tmp.path().join("pipeline").join(format!( "{}.sqlite",