diff --git a/src/cli.rs b/src/cli.rs index ad0f85d8..756df011 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -737,6 +737,34 @@ pub enum MemoryAction { #[derive(Subcommand)] pub enum MigrateAction { + /// Consolidate two non-empty profile shards for one repository identity. + #[command( + long_about = CONSOLIDATE_LONG_ABOUT, + after_help = CONSOLIDATE_AFTER_HELP + )] + Consolidate { + /// Repository path whose git-common-dir identity owns both shards. + #[arg(long, default_value = ".")] + project: String, + /// Legacy/input project id to preserve and merge. + #[arg(long = "source-project-id")] + source_project_id: String, + /// Currently selected project id to use as the merge base. + #[arg(long = "target-project-id")] + target_project_id: String, + /// Profile root containing projects/ shards. + #[arg(long = "profile-root")] + profile_root: Option, + /// Apply the planned consolidation. Omit for a read-only inventory. + #[arg(long)] + apply: bool, + /// Confirmation token printed by the read-only inventory. + #[arg(long = "confirm-token", requires = "apply")] + confirm_token: Option, + /// Output as JSON. + #[arg(long)] + json: bool, + }, /// Build a readonly migration inventory or manifest plan Plan { /// Root directory to scan (repeatable). Defaults to the current directory. diff --git a/src/cli/help.rs b/src/cli/help.rs index 93c28119..ed1b7248 100644 --- a/src/cli/help.rs +++ b/src/cli/help.rs @@ -503,7 +503,8 @@ tracedecay dashboard (review UI), tracedecay memory curate."; pub(crate) const MIGRATE_LONG_ABOUT: &str = "\ Plans and executes profile-storage migrations: inventory scans, manifest \ plans, staged copy + cutover, verification, rollback, and registry cleanup. \ -Mutating steps require the confirmation token printed by `migrate plan`; \ +Explicit split-store consolidation preserves both inputs and builds a new \ +shard before marker cutover. Mutating steps require a dry-run confirmation token; \ start with plan/verify, which never touch source stores."; pub(crate) const MIGRATE_AFTER_HELP: &str = "\ @@ -512,10 +513,29 @@ Examples: tracedecay migrate plan --manifest plan.json Write a manifest plan tracedecay migrate verify --manifest plan.json Check without mutating tracedecay migrate apply --manifest plan.json --confirm-token + tracedecay migrate consolidate --source-project-id --target-project-id tracedecay migrate registry-gc Dry-run stale-registry cleanup Related: tracedecay projects (registry view), tracedecay wipe."; +pub(crate) const CONSOLIDATE_LONG_ABOUT: &str = "\ +Consolidates exactly two profile shards that belong to one git-common-dir identity. \ +The default is a read-only dry-run: it freezes and inventories both inputs, reports \ +collision semantics, and prints an immutable confirmation token. The complete workflow \ +is offline-only: stop the daemon and all MCP/CLI writers first. Apply builds and verifies a third shard; \ +the two originals and complete backups remain preserved. Registry and repository markers \ +move only after verification succeeds."; + +pub(crate) const CONSOLIDATE_AFTER_HELP: &str = "\ +Dry-run: + tracedecay migrate consolidate --project . --source-project-id --target-project-id + +Apply the exact frozen plan: + tracedecay migrate consolidate --project . --source-project-id --target-project-id --apply --confirm-token + +Stop the TraceDecay daemon and all MCP/CLI writers before the dry-run. Rerun it if \ +either input changes; the prior token will be rejected."; + pub(crate) const WIPE_LONG_ABOUT: &str = "\ Deletes .tracedecay stores (code graph, memory, sessions) for the current \ folder, its parents, and its children — or every tracked project with --all. \ diff --git a/src/cli/parse_tests.rs b/src/cli/parse_tests.rs index a2472cf4..54948d63 100644 --- a/src/cli/parse_tests.rs +++ b/src/cli/parse_tests.rs @@ -1367,6 +1367,46 @@ fn project_selector_flags_parse_for_cli_read_surfaces() { #[test] fn migrate_commands_parse_manifest_scaffolding_flags() { + let consolidate = Cli::try_parse_from([ + "tracedecay", + "migrate", + "consolidate", + "--project", + "/tmp/project", + "--profile-root", + "/tmp/profile", + "--source-project-id", + "proj_old", + "--target-project-id", + "proj_current", + "--apply", + "--confirm-token", + "confirm-123", + "--json", + ]) + .expect("migrate consolidate should parse"); + assert!(matches!( + consolidate.command, + Some(Commands::Migrate { + action: + MigrateAction::Consolidate { + project, + profile_root, + source_project_id, + target_project_id, + apply, + confirm_token, + json, + } + }) if project == "/tmp/project" + && profile_root.as_deref() == Some("/tmp/profile") + && source_project_id == "proj_old" + && target_project_id == "proj_current" + && apply + && confirm_token.as_deref() == Some("confirm-123") + && json + )); + let plan = Cli::try_parse_from([ "tracedecay", "migrate", diff --git a/src/commands.rs b/src/commands.rs index 01dbcf53..a4d4706c 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -66,6 +66,68 @@ fn read_llm_ops_payload(source: &str) -> tracedecay::errors::Result tracedecay::errors::Result<()> { match action { + MigrateAction::Consolidate { + project, + source_project_id, + target_project_id, + profile_root, + apply, + confirm_token, + json, + } => { + let profile_root = profile_root.map_or_else( + || { + tracedecay::config::user_data_dir().ok_or_else(|| { + tracedecay::errors::TraceDecayError::Config { + message: "could not determine TraceDecay profile root".to_string(), + } + }) + }, + |value| Ok(PathBuf::from(value)), + )?; + let options = tracedecay::migrate::consolidate::ConsolidationOptions { + project_root: PathBuf::from(project), + profile_root, + source_project_id, + target_project_id, + }; + let report = if apply { + let token = + confirm_token.ok_or_else(|| tracedecay::errors::TraceDecayError::Config { + message: "--confirm-token is required with --apply".to_string(), + })?; + tracedecay::migrate::consolidate::apply(&options, &token).await? + } else { + tracedecay::migrate::consolidate::plan(&options).await? + }; + if json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + println!("Migration: {}", report.migration_id); + println!("State: {:?}", report.state); + println!( + "Source: {} ({})", + report.source.project_id, + report.source.data_root.display() + ); + println!( + "Target: {} ({})", + report.target.project_id, + report.target.data_root.display() + ); + println!( + "Destination: {} ({})", + report.destination_project_id, + report.destination_data_root.display() + ); + println!("Backups: {}", report.backup_root.display()); + println!("Ledger: {}", report.ledger_path.display()); + if report.dry_run { + println!("Confirmation token: {}", report.confirmation_token); + println!("No files changed."); + } + } + } MigrateAction::Plan { roots, include_all_registered, diff --git a/src/doctor/tests.rs b/src/doctor/tests.rs index b0bb217b..384da0f7 100644 --- a/src/doctor/tests.rs +++ b/src/doctor/tests.rs @@ -353,7 +353,15 @@ async fn current_project_store_surfaces_split_identity_conflict() assert!(diagnostic.contains("proj_doctor_selected"), "{diagnostic}"); assert!(diagnostic.contains("proj_doctor_legacy"), "{diagnostic}"); assert!( - diagnostic.contains("backup-and-consolidate migration"), + diagnostic.contains("tracedecay migrate consolidate"), + "{diagnostic}" + ); + assert!( + diagnostic.contains("--source-project-id proj_doctor_legacy"), + "{diagnostic}" + ); + assert!( + diagnostic.contains("--target-project-id proj_doctor_selected"), "{diagnostic}" ); assert!(diagnostic.contains("no files changed"), "{diagnostic}"); diff --git a/src/lib.rs b/src/lib.rs index f750bd82..0882de27 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,6 +58,7 @@ pub mod mcp; pub mod memory; pub mod migrate; pub mod monitor; +mod open_store_holders; mod path_scope; mod path_tree; pub mod project_registry; @@ -70,6 +71,7 @@ pub mod serde_util; pub mod serve; pub mod sessions; mod shell; +mod sqlite_read_snapshot; pub mod storage; pub mod sync; pub mod text; diff --git a/src/lifecycle_lease.rs b/src/lifecycle_lease.rs index 64d9c74d..3c1a75e3 100644 --- a/src/lifecycle_lease.rs +++ b/src/lifecycle_lease.rs @@ -47,6 +47,22 @@ pub fn acquire_exclusive(operation: &str) -> Result { acquire_exclusive_at(&lifecycle_lock_path()?, operation) } +/// Acquires the lifecycle lease rooted in an explicit profile. Migration +/// commands use this instead of ambient HOME so synthetic profiles and +/// user-selected profile roots cannot accidentally lock a different store. +pub fn acquire_exclusive_for_profile( + profile_root: &Path, + operation: &str, +) -> Result { + std::fs::create_dir_all(profile_root).map_err(|error| TraceDecayError::Config { + message: format!( + "failed to create TraceDecay profile root '{}': {error}", + profile_root.display() + ), + })?; + acquire_exclusive_at(&profile_root.join(LIFECYCLE_LOCK_FILENAME), operation) +} + pub fn acquire_shared(operation: &str) -> Result { acquire_shared_at(&lifecycle_lock_path()?, operation) } diff --git a/src/main.rs b/src/main.rs index 0c9029ea..83d1fda5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -796,7 +796,8 @@ fn should_skip_agent_install_maintenance(command: &Commands) -> bool { // update-plugin contract. // - `Uninstall`: about to remove agent configs — don't reinstall them // first (per the original #84 intent). - // - `Doctor` / `Migrate`: read-only diagnostics — must not mutate agent + // - `Doctor` / `Migrate`: diagnostics and explicitly confirmed storage + // maintenance manage their own lifecycle and must not mutate agent // configs as a side effect. // - `Tool`: per-invocation tool calls are a hot-ish path; skip the // reinstall scan there too. diff --git a/src/migrate/consolidate/evidence.rs b/src/migrate/consolidate/evidence.rs new file mode 100644 index 00000000..242f8016 --- /dev/null +++ b/src/migrate/consolidate/evidence.rs @@ -0,0 +1,194 @@ +use super::*; + +pub(super) struct InputReadEvidence { + pub(super) source_graph: GraphStoreEvidence, + pub(super) target_graph: GraphStoreEvidence, + pub(super) sessions: crate::sqlite_read_snapshot::SnapshotSet, + pub(super) session_fingerprints: BTreeMap, +} + +pub(super) struct GraphStoreEvidence { + pub(super) identities: sqlite::GraphLogicalIdentities, + pub(super) fingerprints: BTreeMap, + generations: BTreeMap, + #[cfg(test)] + peak_scratch_bytes: u64, +} + +impl InputReadEvidence { + pub(super) fn validate( + &self, + source_graphs: &[PathBuf], + target_graphs: &[PathBuf], + session_paths: &[PathBuf], + ) -> Result<()> { + self.source_graph.validate(source_graphs)?; + self.target_graph.validate(target_graphs)?; + self.sessions + .validate_sources_unchanged() + .map_err(io_error)?; + for path in session_paths { + self.sessions.get(path).map_err(io_error)?; + if !self.session_fingerprints.contains_key(path) { + return Err(config_error( + "session database set changed after inspection", + )); + } + } + Ok(()) + } + + pub(super) fn validate_content( + &self, + source_graphs: &[PathBuf], + target_graphs: &[PathBuf], + session_paths: &[PathBuf], + ) -> Result<()> { + self.source_graph.validate_content(source_graphs)?; + self.target_graph.validate_content(target_graphs)?; + validate_path_set( + self.session_fingerprints.keys(), + session_paths, + "session database set changed after inspection", + )?; + for path in session_paths { + let expected = self + .session_fingerprints + .get(path) + .ok_or_else(|| config_error("session database set changed after inspection"))?; + let current = + crate::sqlite_read_snapshot::family_fingerprint(path).map_err(io_error)?; + if ¤t != expected { + return Err(config_error(format!( + "SQLite database family '{}' content changed after inspection", + path.display() + ))); + } + } + Ok(()) + } + + #[cfg(test)] + pub(super) fn retained_database_count(&self) -> usize { + self.sessions.database_count() + } + + #[cfg(test)] + pub(super) fn peak_graph_scratch_bytes(&self) -> u64 { + self.source_graph + .peak_scratch_bytes + .max(self.target_graph.peak_scratch_bytes) + } +} + +impl GraphStoreEvidence { + fn validate(&self, expected_paths: &[PathBuf]) -> Result<()> { + validate_path_set( + self.fingerprints.keys(), + expected_paths, + "graph database set changed after inspection", + )?; + for generation in self.generations.values() { + generation.validate().map_err(io_error)?; + } + Ok(()) + } + + fn validate_content(&self, expected_paths: &[PathBuf]) -> Result<()> { + validate_path_set( + self.fingerprints.keys(), + expected_paths, + "graph database set changed after inspection", + )?; + for path in expected_paths { + let expected = self + .fingerprints + .get(path) + .ok_or_else(|| config_error("graph database set changed after inspection"))?; + let current = + crate::sqlite_read_snapshot::family_fingerprint(path).map_err(io_error)?; + if ¤t != expected { + return Err(config_error(format!( + "SQLite database family '{}' content changed after inspection", + path.display() + ))); + } + } + Ok(()) + } +} + +fn validate_path_set<'a>( + actual: impl Iterator, + expected: &[PathBuf], + message: &str, +) -> Result<()> { + let actual = actual.cloned().collect::>(); + let expected = expected.iter().cloned().collect::>(); + if actual == expected { + Ok(()) + } else { + Err(config_error(message)) + } +} + +pub(super) async fn capture_input_evidence( + source_graphs: &[PathBuf], + target_graphs: &[PathBuf], + session_paths: &[PathBuf], + scratch_root: &Path, +) -> Result { + let source_graph = capture_graph_evidence(source_graphs, scratch_root).await?; + let target_graph = capture_graph_evidence(target_graphs, scratch_root).await?; + let sessions = + crate::sqlite_read_snapshot::SnapshotSet::capture_in(session_paths, scratch_root) + .await + .map_err(io_error)?; + let mut session_fingerprints = BTreeMap::new(); + for path in session_paths { + sessions.get(path).map_err(io_error)?; + let fingerprint = + crate::sqlite_read_snapshot::family_fingerprint(path).map_err(io_error)?; + session_fingerprints.insert(path.clone(), fingerprint); + } + sessions.validate_sources_unchanged().map_err(io_error)?; + Ok(InputReadEvidence { + source_graph, + target_graph, + sessions, + session_fingerprints, + }) +} + +async fn capture_graph_evidence( + paths: &[PathBuf], + scratch_root: &Path, +) -> Result { + let mut identities = sqlite::GraphLogicalIdentities::default(); + let mut fingerprints = BTreeMap::new(); + let mut generations = BTreeMap::new(); + #[cfg(test)] + let mut peak_scratch_bytes = 0_u64; + for path in paths { + let snapshot = crate::sqlite_read_snapshot::open_in(path, scratch_root) + .await + .map_err(io_error)?; + #[cfg(test)] + { + peak_scratch_bytes = peak_scratch_bytes.max(snapshot.copied_bytes()); + } + sqlite::extend_graph_identities(snapshot.connection(), &mut identities).await?; + let fingerprint = + crate::sqlite_read_snapshot::family_fingerprint(path).map_err(io_error)?; + snapshot.validate_source().map_err(io_error)?; + generations.insert(path.clone(), snapshot.source_generation()); + fingerprints.insert(path.clone(), fingerprint); + } + Ok(GraphStoreEvidence { + identities, + fingerprints, + generations, + #[cfg(test)] + peak_scratch_bytes, + }) +} diff --git a/src/migrate/consolidate/files.rs b/src/migrate/consolidate/files.rs new file mode 100644 index 00000000..517fcab1 --- /dev/null +++ b/src/migrate/consolidate/files.rs @@ -0,0 +1,206 @@ +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +use super::{config_error, io_error}; +use crate::errors::Result; +use crate::storage; + +pub(super) fn relative_file_map(root: &Path) -> Result> { + let mut files = BTreeMap::new(); + collect_files(root, root, &mut files)?; + Ok(files) +} + +fn collect_files( + root: &Path, + current: &Path, + files: &mut BTreeMap, +) -> Result<()> { + let mut entries = fs::read_dir(current) + .map_err(io_error)? + .collect::>>() + .map_err(io_error)?; + entries.sort_by_key(fs::DirEntry::file_name); + for entry in entries { + let path = entry.path(); + let metadata = path.symlink_metadata().map_err(io_error)?; + if metadata.file_type().is_symlink() { + return Err(config_error(format!( + "profile store artifact '{}' is a symlink; refusing unsafe traversal", + path.display() + ))); + } + if metadata.is_dir() { + collect_files(root, &path, files)?; + } else if metadata.is_file() { + let relative = path + .strip_prefix(root) + .map_err(|error| config_error(error.to_string()))? + .to_path_buf(); + files.insert(relative, path); + } + } + Ok(()) +} + +pub(super) fn tree_stats(root: &Path) -> Result<(usize, u64)> { + let files = relative_file_map(root)?; + let mut bytes = 0_u64; + for path in files.values() { + bytes = bytes.saturating_add(fs::metadata(path).map_err(io_error)?.len()); + } + Ok((files.len(), bytes)) +} + +pub(super) fn copy_tree_exact(source: &Path, target: &Path) -> Result<()> { + for (relative, path) in relative_file_map(source)? { + copy_file_exact(&path, &target.join(relative))?; + } + Ok(()) +} + +pub(super) fn copy_file_exact(source: &Path, target: &Path) -> Result<()> { + if target.exists() { + if file_digest(source)? == file_digest(target)? { + sync_file_and_parent(target)?; + return Ok(()); + } + return Err(config_error(format!( + "existing migration artifact '{}' differs from source '{}'", + target.display(), + source.display() + ))); + } + copy_file_atomic(source, target)?; + if file_digest(source)? != file_digest(target)? { + let _ = fs::remove_file(target); + return Err(config_error(format!( + "copied migration artifact '{}' failed checksum verification against '{}'", + target.display(), + source.display() + ))); + } + Ok(()) +} + +pub(super) fn copy_file_atomic(source: &Path, target: &Path) -> Result<()> { + let parent = target + .parent() + .ok_or_else(|| config_error("artifact target has no parent"))?; + fs::create_dir_all(parent).map_err(io_error)?; + let temp = target.with_extension(format!("tmp-{}", std::process::id())); + fs::copy(source, &temp).map_err(io_error)?; + File::open(&temp) + .and_then(|file| file.sync_all()) + .map_err(io_error)?; + fs::rename(&temp, target).map_err(io_error)?; + sync_parent_directory(parent)?; + Ok(()) +} + +fn sync_file_and_parent(path: &Path) -> Result<()> { + File::open(path) + .and_then(|file| file.sync_all()) + .map_err(io_error)?; + let parent = path + .parent() + .ok_or_else(|| config_error("artifact target has no parent"))?; + sync_parent_directory(parent) +} + +#[cfg(unix)] +pub(super) fn sync_parent_directory(parent: &Path) -> Result<()> { + File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(io_error) +} + +#[cfg(not(unix))] +pub(super) fn sync_parent_directory(_parent: &Path) -> Result<()> { + // Windows directory handles commonly reject sync_all with AccessDenied. + // File data is still flushed before the atomic rename. + Ok(()) +} + +pub(super) fn copy_sqlite_family_exact(source: &Path, target: &Path) -> Result<()> { + copy_file_exact(source, target)?; + for suffix in ["-wal", "-shm"] { + let source_sidecar = sqlite_sidecar(source, suffix); + if source_sidecar.is_file() { + copy_file_exact(&source_sidecar, &sqlite_sidecar(target, suffix))?; + } + } + Ok(()) +} + +pub(super) fn sqlite_sidecar(path: &Path, suffix: &str) -> PathBuf { + let mut value = path.as_os_str().to_os_string(); + value.push(suffix); + PathBuf::from(value) +} + +pub(super) fn remove_runtime_files(root: &Path) -> Result<()> { + for relative in ["sync.lock", ".branch-add.lock", ".dirty"] { + let path = root.join(relative); + match fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(io_error(error)), + } + } + Ok(()) +} + +pub(super) fn excluded_source_artifact(relative: &Path) -> bool { + let value = relative.to_string_lossy(); + value == crate::config::DB_FILENAME + || value == storage::SESSIONS_DB_FILENAME + || value == storage::BRANCH_META_FILENAME + || value == storage::STORE_MANIFEST_FILENAME + || value.starts_with("branches/") + || value.ends_with("-wal") + || value.ends_with("-shm") + || is_runtime_lock(relative) +} + +pub(super) fn is_runtime_lock(relative: &Path) -> bool { + matches!( + relative.file_name().and_then(|value| value.to_str()), + Some("sync.lock" | ".branch-add.lock" | ".dirty") + ) +} + +pub(super) fn is_sqlite_sidecar(relative: &Path) -> bool { + relative + .file_name() + .and_then(|value| value.to_str()) + .is_some_and(|value| value.ends_with("-shm") || value.ends_with("-wal")) +} + +pub(super) fn is_sqlite_database(relative: &Path) -> bool { + relative + .extension() + .is_some_and(|value| value.eq_ignore_ascii_case("db")) +} + +pub(super) fn is_reference_artifact(relative: &Path) -> bool { + relative.starts_with("lcm-payloads") || relative.starts_with("response-handles") +} + +pub(super) fn file_digest(path: &Path) -> Result<[u8; 32]> { + let mut file = File::open(path).map_err(io_error)?; + let mut hash = Sha256::new(); + let mut buffer = vec![0_u8; 64 * 1024].into_boxed_slice(); + loop { + let read = file.read(&mut buffer).map_err(io_error)?; + if read == 0 { + break; + } + hash.update(&buffer[..read]); + } + Ok(hash.finalize().into()) +} diff --git a/src/migrate/consolidate/finalize.rs b/src/migrate/consolidate/finalize.rs new file mode 100644 index 00000000..fcc75b9a --- /dev/null +++ b/src/migrate/consolidate/finalize.rs @@ -0,0 +1,192 @@ +use super::*; + +pub(super) async fn verify_destination( + resolved: &ResolvedPlan, + session_offsets: &sqlite::SessionMergeOffsets, +) -> Result<()> { + let destination = &resolved.report.destination_data_root; + let graph = destination.join(crate::config::DB_FILENAME); + let sessions = destination.join(storage::SESSIONS_DB_FILENAME); + let meta = branch_meta::load_branch_meta(destination) + .ok_or_else(|| config_error("consolidated branch metadata is invalid"))?; + let expected_branches = resolved + .target_meta + .branches + .len() + .saturating_add(resolved.source_meta.branches.len()); + if meta.branches.len() != expected_branches { + return Err(config_error(format!( + "destination has {} branch graph(s), expected {expected_branches}", + meta.branches.len() + ))); + } + let destination_graphs = graph_db_paths_for_root(destination, &meta)?; + let mut destination_identities = sqlite::GraphLogicalIdentities::default(); + for path in &destination_graphs { + let snapshot = crate::sqlite_read_snapshot::open_in(path, resolved.scratch_root.path()) + .await + .map_err(io_error)?; + sqlite::quick_check_connection(snapshot.connection(), path).await?; + if path == &graph { + sqlite::extend_graph_identities(snapshot.connection(), &mut destination_identities) + .await?; + } + snapshot.validate_source().map_err(io_error)?; + } + if !resolved + .evidence + .source_graph + .identities + .facts_union_matches( + &resolved.evidence.target_graph.identities, + &destination_identities, + ) + { + return Err(config_error( + "destination fact logical union differs from frozen inputs", + )); + } + if !resolved + .evidence + .source_graph + .identities + .feedback_union_matches( + &resolved.evidence.target_graph.identities, + &destination_identities, + ) + { + return Err(config_error( + "destination feedback logical union differs from frozen inputs", + )); + } + let destination_snapshots = crate::sqlite_read_snapshot::SnapshotSet::capture_in( + std::slice::from_ref(&sessions), + resolved.scratch_root.path(), + ) + .await + .map_err(io_error)?; + sqlite::quick_check_in(&destination_snapshots, &sessions).await?; + let input_root = destination.join(INPUT_DIR); + let source_input = input_root.join("source-sessions.db"); + let target_input = input_root.join("target-sessions.db"); + let input_snapshots = crate::sqlite_read_snapshot::SnapshotSet::capture_in( + &[source_input.clone(), target_input.clone()], + resolved.scratch_root.path(), + ) + .await + .map_err(io_error)?; + sqlite::verify_session_union_sql( + &input_snapshots, + &source_input, + &target_input, + &destination_snapshots, + &sessions, + destination, + session_offsets, + ) + .await?; + Ok(()) +} + +pub(super) async fn register_destination(resolved: &ResolvedPlan) -> Result<()> { + let global_path = resolved + .report + .destination_data_root + .parent() + .and_then(Path::parent) + .ok_or_else(|| config_error("destination shard has no profile root"))? + .join("global.db"); + let db = GlobalDb::open_at(&global_path) + .await + .ok_or_else(|| config_error("could not open global registry for consolidation"))?; + let project = db + .upsert_code_project( + &resolved.report.destination_project_id, + &resolved.report.project_root, + Some(&resolved.report.git_common_dir), + git_remote_url(&resolved.report.project_root).as_deref(), + Some(&resolved.target_meta.default_branch), + ) + .await + .ok_or_else(|| config_error("could not register consolidated project"))?; + db.upsert_project_alias(&resolved.report.project_root, &project.project_id) + .await + .ok_or_else(|| config_error("could not register consolidated project alias"))?; + let store_id = format!("store:{}:profile_sharded", project.project_id); + let store_relpath = format!("projects/{}", project.project_id); + let now = crate::tracedecay::current_timestamp(); + db.upsert_store_instance(StoreInstanceUpsert { + store_id: store_id.clone(), + project_id: project.project_id.clone(), + store_kind: "code_project".to_string(), + storage_mode: "profile_sharded".to_string(), + store_relpath: store_relpath.clone(), + manifest_relpath: Some(format!( + "{store_relpath}/{}", + storage::STORE_MANIFEST_FILENAME + )), + last_verified_at: Some(now), + last_write_at: Some(now), + }) + .await + .ok_or_else(|| config_error("could not register consolidated store"))?; + + let meta = branch_meta::load_branch_meta(&resolved.report.destination_data_root) + .ok_or_else(|| config_error("consolidated branch metadata disappeared"))?; + for (branch_name, entry) in &meta.branches { + db.upsert_graph_scope(GraphScopeUpsert { + graph_scope_id: format!("{store_id}:branch:{branch_name}"), + project_id: project.project_id.clone(), + store_id: store_id.clone(), + branch_name: branch_name.clone(), + db_relpath: format!("{store_relpath}/{}", entry.db_file), + parent_scope_id: entry + .parent + .as_deref() + .map(|parent| format!("{store_id}:branch:{parent}")), + last_synced_at: entry.last_synced_at.parse().ok(), + writable: true, + }) + .await + .ok_or_else(|| config_error("could not register consolidated graph scope"))?; + } + for (kind, relative) in [ + ("graph_db", crate::config::DB_FILENAME), + ("sessions_db", storage::SESSIONS_DB_FILENAME), + ("branch_meta", storage::BRANCH_META_FILENAME), + ("store_manifest", storage::STORE_MANIFEST_FILENAME), + ] { + let path = resolved.report.destination_data_root.join(relative); + db.upsert_store_artifact(StoreArtifactUpsert { + store_id: store_id.clone(), + artifact_kind: kind.to_string(), + relpath: format!("{store_relpath}/{relative}"), + size_bytes: fs::metadata(path) + .ok() + .and_then(|meta| i64::try_from(meta.len()).ok()), + schema_version: (kind == "store_manifest") + .then(|| storage::STORE_MANIFEST_SCHEMA_VERSION.to_string()), + updated_at: Some(now), + }) + .await + .ok_or_else(|| config_error("could not register consolidated artifact"))?; + } + db.checkpoint().await; + db.close(); + Ok(()) +} + +pub(super) fn cut_over_markers(resolved: &ResolvedPlan) -> Result<()> { + storage::write_enrollment_marker( + &resolved.report.project_root, + &EnrollmentMarker { + project_id: resolved.report.destination_project_id.clone(), + storage_mode: StorageMode::ProfileSharded, + }, + )?; + storage::write_repository_identity_marker( + &resolved.report.project_root, + &resolved.report.destination_project_id, + )?; + Ok(()) +} diff --git a/src/migrate/consolidate/mod.rs b/src/migrate/consolidate/mod.rs new file mode 100644 index 00000000..61724e98 --- /dev/null +++ b/src/migrate/consolidate/mod.rs @@ -0,0 +1,1007 @@ +//! Explicit, offline consolidation of two profile shards for one repository. +//! +//! The runtime deliberately refuses to guess when both shards contain data. +//! This workflow builds a third deterministic shard, preserving both inputs +//! and cutting the repository marker over only after the new shard and global +//! registry have verified successfully. + +mod evidence; +mod files; +mod finalize; +mod preflight; +mod prepare; +mod sqlite; + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use evidence::{GraphStoreEvidence, InputReadEvidence, capture_input_evidence}; +#[cfg(test)] +use files::sqlite_sidecar; +use files::{ + copy_file_atomic, copy_sqlite_family_exact, copy_tree_exact, excluded_source_artifact, + file_digest, is_reference_artifact, is_runtime_lock, is_sqlite_database, is_sqlite_sidecar, + relative_file_map, tree_stats, +}; +use finalize::{cut_over_markers, register_destination, verify_destination}; +use preflight::{acquire_store_locks, ensure_profile_offline, preflight_disk_space}; +use prepare::prepare_destination; + +use crate::branch_meta::{self, BranchMeta}; +use crate::errors::{Result, TraceDecayError}; +use crate::global_db::{GlobalDb, GraphScopeUpsert, StoreArtifactUpsert, StoreInstanceUpsert}; +use crate::storage::{ + self, EnrollmentMarker, PrivateStoreIo, StorageMode, StoreKind, StoreLayout, StoreManifest, +}; + +const LEDGER_SCHEMA_VERSION: u32 = 1; +const BACKUP_DIR: &str = "migration-backups"; +const LEDGER_DIR: &str = "migration-inventory"; +const PRESERVED_DIR: &str = "consolidation-preserved"; +const INPUT_DIR: &str = ".consolidation-input"; + +#[derive(Debug, Clone)] +pub struct ConsolidationOptions { + pub project_root: PathBuf, + pub profile_root: PathBuf, + pub source_project_id: String, + pub target_project_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ConsolidationState { + Planned, + BackupsReady, + DestinationReady, + DatabasesMerged, + ArtifactsMerged, + Registered, + Applied, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StoreInventory { + pub project_id: String, + pub data_root: PathBuf, + pub graph_databases: usize, + pub facts: u64, + pub feedback_events: u64, + pub sessions: u64, + pub messages: u64, + pub lcm_raw_messages: u64, + pub branches: usize, + pub artifact_files: usize, + pub bytes: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CollisionSummary { + pub fact_content_overlaps: u64, + pub session_overlaps: u64, + pub message_overlaps: u64, + pub lcm_message_overlaps: u64, + pub artifact_path_overlaps: usize, + pub differing_artifact_paths: Vec, + pub semantics: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConsolidationReport { + pub migration_id: String, + pub state: ConsolidationState, + pub project_root: PathBuf, + pub git_common_dir: PathBuf, + pub source: StoreInventory, + pub target: StoreInventory, + pub destination_project_id: String, + pub destination_data_root: PathBuf, + pub backup_root: PathBuf, + pub ledger_path: PathBuf, + pub confirmation_token: String, + pub collisions: CollisionSummary, + pub dry_run: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ConsolidationLedger { + schema_version: u32, + migration_id: String, + confirmation_token: String, + input_fingerprint: String, + source_project_id: String, + target_project_id: String, + destination_project_id: String, + project_root: PathBuf, + git_common_dir: PathBuf, + state: ConsolidationState, + graph_offsets: Vec, + session_offsets: Option, + preserved_collisions: Vec, +} + +struct ResolvedPlan { + report: ConsolidationReport, + input_fingerprint: String, + source_layout: StoreLayout, + target_layout: StoreLayout, + source_meta: BranchMeta, + target_meta: BranchMeta, + scratch_root: MigrationScratchRoot, + evidence: Arc, +} + +static NEXT_MIGRATION_SCRATCH: AtomicU64 = AtomicU64::new(0); + +struct MigrationScratchRoot { + path: PathBuf, +} + +impl MigrationScratchRoot { + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for MigrationScratchRoot { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +pub async fn plan(options: &ConsolidationOptions) -> Result { + ensure_profile_offline(options)?; + Ok(resolve_plan(options).await?.report) +} + +pub async fn apply( + options: &ConsolidationOptions, + confirmation_token: &str, +) -> Result { + apply_with_stop(options, confirmation_token, None).await +} + +async fn apply_with_stop( + options: &ConsolidationOptions, + confirmation_token: &str, + stop_after: Option, +) -> Result { + apply_with_faults(options, confirmation_token, stop_after, None).await +} + +#[cfg(test)] +async fn apply_with_prepare_stop( + options: &ConsolidationOptions, + confirmation_token: &str, + prepare_stop: prepare::PrepareStop, +) -> Result { + apply_with_faults(options, confirmation_token, None, Some(prepare_stop)).await +} + +async fn apply_with_faults( + options: &ConsolidationOptions, + confirmation_token: &str, + stop_after: Option, + prepare_stop: Option, +) -> Result { + ensure_profile_offline(options)?; + let _lifecycle = crate::lifecycle_lease::acquire_exclusive_for_profile( + &options.profile_root, + "profile shard consolidation", + )?; + let resolved = resolve_plan_allowing_applied(options).await?; + if resolved.report.confirmation_token != confirmation_token { + return Err(config_error(format!( + "confirmation token mismatch; rerun the dry-run and pass --confirm-token {}", + resolved.report.confirmation_token + ))); + } + preflight_disk_space(&resolved)?; + let _store_locks = acquire_store_locks(&resolved.source_layout, &resolved.target_layout)?; + let guarded_paths = input_database_paths(&resolved)?; + let source_graphs = graph_db_paths(&resolved.source_layout, &resolved.source_meta)?; + let target_graphs = graph_db_paths(&resolved.target_layout, &resolved.target_meta)?; + let session_paths = vec![ + resolved.source_layout.sessions_db_path.clone(), + resolved.target_layout.sessions_db_path.clone(), + ]; + resolved + .evidence + .validate(&source_graphs, &target_graphs, &session_paths)?; + let _database_guards = sqlite::acquire_offline_guards(&guarded_paths).await?; + // Advisory store locks do not cover old or direct MCP writers. Recompute + // under SQLite write reservations so the token and backups describe the + // exact frozen inputs used below. + let locked = resolve_plan_inner(options, true, Some(Arc::clone(&resolved.evidence))).await?; + if locked.report.confirmation_token != confirmation_token { + return Err(config_error(format!( + "input stores changed after the dry-run; rerun it and pass --confirm-token {}", + locked.report.confirmation_token + ))); + } + let resolved = locked; + preflight_disk_space(&resolved)?; + let ledger_path = resolved.report.ledger_path.clone(); + let mut ledger = load_or_create_ledger(&resolved, &ledger_path)?; + validate_ledger(&ledger, &resolved)?; + if ledger.state == ConsolidationState::Applied { + let mut report = resolved.report; + report.state = ConsolidationState::Applied; + report.dry_run = false; + return Ok(report); + } + + if ledger.state == ConsolidationState::Planned { + backup_store(&resolved.source_layout, &resolved.report.backup_root)?; + backup_store(&resolved.target_layout, &resolved.report.backup_root)?; + ledger.state = ConsolidationState::BackupsReady; + save_ledger(&ledger_path, &ledger)?; + maybe_stop(&ledger.state, stop_after.as_ref())?; + } + + if ledger.state == ConsolidationState::BackupsReady { + if prepare_stop.is_some() { + prepare::prepare_destination_with_stop(&resolved, prepare_stop)?; + } else { + prepare_destination(&resolved)?; + } + ledger.state = ConsolidationState::DestinationReady; + save_ledger(&ledger_path, &ledger)?; + maybe_stop(&ledger.state, stop_after.as_ref())?; + } + + if ledger.state == ConsolidationState::DestinationReady { + merge_databases(&resolved, &mut ledger).await?; + ledger.state = ConsolidationState::DatabasesMerged; + save_ledger(&ledger_path, &ledger)?; + maybe_stop(&ledger.state, stop_after.as_ref())?; + } + + if ledger.state == ConsolidationState::DatabasesMerged { + merge_non_database_artifacts(&resolved, &mut ledger)?; + write_destination_manifest(&resolved)?; + let session_offsets = ledger + .session_offsets + .as_ref() + .ok_or_else(|| config_error("session merge offsets are missing from the ledger"))?; + verify_destination(&resolved, session_offsets).await?; + ledger.state = ConsolidationState::ArtifactsMerged; + save_ledger(&ledger_path, &ledger)?; + maybe_stop(&ledger.state, stop_after.as_ref())?; + } + + if ledger.state == ConsolidationState::ArtifactsMerged { + remove_verification_inputs(&resolved)?; + register_destination(&resolved).await?; + ledger.state = ConsolidationState::Registered; + save_ledger(&ledger_path, &ledger)?; + maybe_stop(&ledger.state, stop_after.as_ref())?; + } + + if ledger.state == ConsolidationState::Registered { + cut_over_markers(&resolved)?; + ledger.state = ConsolidationState::Applied; + save_ledger(&ledger_path, &ledger)?; + } + + let mut report = resolved.report; + report.state = ledger.state; + report.dry_run = false; + Ok(report) +} + +fn maybe_stop(state: &ConsolidationState, stop_after: Option<&ConsolidationState>) -> Result<()> { + if stop_after == Some(state) { + return Err(config_error(format!( + "synthetic interruption after {state:?}" + ))); + } + Ok(()) +} + +async fn resolve_plan(options: &ConsolidationOptions) -> Result { + resolve_plan_inner(options, false, None).await +} + +async fn resolve_plan_allowing_applied(options: &ConsolidationOptions) -> Result { + resolve_plan_inner(options, true, None).await +} + +async fn resolve_plan_inner( + options: &ConsolidationOptions, + allow_destination_marker: bool, + evidence: Option>, +) -> Result { + storage::validate_project_id(&options.source_project_id).map_err(config_error)?; + storage::validate_project_id(&options.target_project_id).map_err(config_error)?; + if options.source_project_id == options.target_project_id { + return Err(config_error("source and target project ids must differ")); + } + let project_root = options + .project_root + .canonicalize() + .map_err(|error| config_error(format!("could not resolve project root: {error}")))?; + let profile_root = options + .profile_root + .canonicalize() + .map_err(|error| config_error(format!("could not resolve profile root: {error}")))?; + let git_common_dir = crate::worktree::git_common_dir(&project_root) + .ok_or_else(|| config_error("project must be an attached git checkout"))?; + let source_layout = layout_for_id(&project_root, &profile_root, &options.source_project_id)?; + let target_layout = layout_for_id(&project_root, &profile_root, &options.target_project_id)?; + let source_manifest = validate_manifest(&source_layout, &options.source_project_id)?; + let target_manifest = validate_manifest(&target_layout, &options.target_project_id)?; + let destination_project_id = destination_project_id( + &git_common_dir, + &options.source_project_id, + &options.target_project_id, + ); + let repository_marker = storage::read_repository_identity_marker(&project_root)? + .ok_or_else(|| config_error("repository identity marker is required"))?; + let marker_ok = repository_marker.project_id == options.target_project_id + || (allow_destination_marker && repository_marker.project_id == destination_project_id); + if !marker_ok { + return Err(config_error(format!( + "target project id '{}' is not the repository-selected shard '{}'", + options.target_project_id, repository_marker.project_id + ))); + } + if !manifest_matches_identity( + &source_manifest, + &target_manifest, + &project_root, + &git_common_dir, + ) || !manifest_matches_identity( + &target_manifest, + &target_manifest, + &project_root, + &git_common_dir, + ) { + return Err(config_error( + "source and target manifests do not prove one exact git-common-dir identity", + )); + } + reject_ambiguous_shards( + options, + &profile_root, + &project_root, + &git_common_dir, + &target_manifest, + &destination_project_id, + )?; + + let source_meta = load_required_branch_meta(&source_layout)?; + let target_meta = load_required_branch_meta(&target_layout)?; + let source_graphs = graph_db_paths(&source_layout, &source_meta)?; + let target_graphs = graph_db_paths(&target_layout, &target_meta)?; + let session_paths = vec![ + source_layout.sessions_db_path.clone(), + target_layout.sessions_db_path.clone(), + ]; + let mut input_paths = source_graphs.clone(); + input_paths.extend(target_graphs.iter().cloned()); + input_paths.extend(session_paths.iter().cloned()); + input_paths.sort(); + input_paths.dedup(); + preflight::ensure_no_open_store_holders(&input_paths)?; + let scratch_root = migration_scratch_root(&profile_root)?; + let evidence = match evidence { + Some(evidence) => { + evidence.validate_content(&source_graphs, &target_graphs, &session_paths)?; + evidence + } + None => Arc::new( + capture_input_evidence( + &source_graphs, + &target_graphs, + &session_paths, + scratch_root.path(), + ) + .await?, + ), + }; + let source = inventory_store( + &evidence.source_graph, + &evidence.sessions, + &source_layout, + &source_meta, + ) + .await?; + let target = inventory_store( + &evidence.target_graph, + &evidence.sessions, + &target_layout, + &target_meta, + ) + .await?; + let collisions = collision_summary(&evidence, &source_layout, &target_layout).await?; + let input_fingerprint = fingerprint_inputs(&evidence, &source_layout, &target_layout)?; + let migration_id = format!("consolidate_{}", &destination_project_id[5..]); + let confirmation_token = confirmation_token(&input_fingerprint, &migration_id); + let destination_data_root = + storage::profile_sharded_data_root(&profile_root, &destination_project_id); + let backup_root = profile_root.join(BACKUP_DIR).join(&migration_id); + let ledger_path = profile_root + .join(LEDGER_DIR) + .join(format!("{migration_id}.json")); + let state = load_ledger(&ledger_path)? + .map(|ledger| ledger.state) + .unwrap_or(ConsolidationState::Planned); + Ok(ResolvedPlan { + report: ConsolidationReport { + migration_id, + state, + project_root, + git_common_dir, + source, + target, + destination_project_id, + destination_data_root, + backup_root, + ledger_path, + confirmation_token, + collisions, + dry_run: true, + }, + input_fingerprint, + source_layout, + target_layout, + source_meta, + target_meta, + scratch_root, + evidence, + }) +} + +fn layout_for_id( + project_root: &Path, + profile_root: &Path, + project_id: &str, +) -> Result { + storage::profile_sharded_layout( + project_root, + profile_root, + &EnrollmentMarker { + project_id: project_id.to_string(), + storage_mode: StorageMode::ProfileSharded, + }, + ) +} + +fn validate_manifest(layout: &StoreLayout, project_id: &str) -> Result { + let path = layout + .manifest_path + .as_ref() + .ok_or_else(|| config_error("profile shard has no store manifest path"))?; + let manifest = storage::read_store_manifest(path)?; + if manifest.project_id.as_deref() != Some(project_id) + || manifest.schema_version != storage::STORE_MANIFEST_SCHEMA_VERSION + || manifest.store_kind != StoreKind::CodeProject + || manifest.storage_mode != StorageMode::ProfileSharded + || !same_path(&manifest.data_root, &layout.data_root) + || !same_path( + &manifest.data_root.join(&manifest.graph_db_relpath), + &layout.graph_db_path, + ) + || !same_path( + &manifest.data_root.join(&manifest.sessions_db_relpath), + &layout.sessions_db_path, + ) + || !same_path( + &manifest.data_root.join(&manifest.branch_meta_relpath), + &layout.branch_meta_path, + ) + { + return Err(config_error(format!( + "store manifest '{}' does not match profile shard '{}'", + path.display(), + layout.data_root.display() + ))); + } + Ok(manifest) +} + +fn manifest_matches_identity( + candidate: &StoreManifest, + selected: &StoreManifest, + project_root: &Path, + git_common_dir: &Path, +) -> bool { + if same_path(&candidate.project_root, project_root) { + return true; + } + if candidate.project_root.is_dir() + && crate::worktree::git_common_dir(&candidate.project_root) + .is_some_and(|path| same_path(&path, git_common_dir)) + { + return true; + } + // A repository move carries its identity marker inside the git common + // directory. If both manifests name the same now-missing former root, + // the marker-selected manifest is the proof that the pair moved together. + !candidate.project_root.exists() + && !selected.project_root.exists() + && same_path(&candidate.project_root, &selected.project_root) +} + +fn reject_ambiguous_shards( + options: &ConsolidationOptions, + profile_root: &Path, + project_root: &Path, + git_common_dir: &Path, + selected: &StoreManifest, + destination_project_id: &str, +) -> Result<()> { + let projects = profile_root.join("projects"); + let mut matches = Vec::new(); + let Ok(entries) = fs::read_dir(projects) else { + return Ok(()); + }; + for entry in entries.flatten() { + let path = entry.path().join(storage::STORE_MANIFEST_FILENAME); + let Ok(manifest) = storage::read_store_manifest(&path) else { + continue; + }; + let Some(project_id) = manifest.project_id.as_deref() else { + continue; + }; + if project_id == destination_project_id { + continue; + } + if manifest_matches_identity(&manifest, selected, project_root, git_common_dir) { + matches.push(project_id.to_string()); + } + } + matches.sort(); + matches.dedup(); + let expected = BTreeSet::from([ + options.source_project_id.clone(), + options.target_project_id.clone(), + ]); + let actual = matches.iter().cloned().collect::>(); + if actual != expected { + return Err(config_error(format!( + "ambiguous split-store identity: expected exactly {expected:?}, found {actual:?}; no files changed" + ))); + } + Ok(()) +} + +fn load_required_branch_meta(layout: &StoreLayout) -> Result { + branch_meta::load_branch_meta(&layout.data_root).ok_or_else(|| { + config_error(format!( + "missing or invalid branch metadata at '{}'", + layout.branch_meta_path.display() + )) + }) +} + +async fn inventory_store( + graph: &GraphStoreEvidence, + sessions: &crate::sqlite_read_snapshot::SnapshotSet, + layout: &StoreLayout, + meta: &BranchMeta, +) -> Result { + let graph_paths = graph_db_paths(layout, meta)?; + let (artifact_files, bytes) = tree_stats(&layout.data_root)?; + Ok(StoreInventory { + project_id: layout.identity.project_id.clone().unwrap_or_default(), + data_root: layout.data_root.clone(), + graph_databases: graph_paths.len(), + facts: graph.identities.fact_count(), + feedback_events: graph.identities.feedback_count(), + sessions: sqlite::count_rows_in(sessions, &layout.sessions_db_path, "sessions").await?, + messages: sqlite::count_rows_in(sessions, &layout.sessions_db_path, "session_messages") + .await?, + lcm_raw_messages: sqlite::count_rows_in( + sessions, + &layout.sessions_db_path, + "lcm_raw_messages", + ) + .await?, + branches: meta.branches.len(), + artifact_files, + bytes, + }) +} + +async fn collision_summary( + evidence: &InputReadEvidence, + source: &StoreLayout, + target: &StoreLayout, +) -> Result { + let source_files = relative_file_map(&source.data_root)?; + let target_files = relative_file_map(&target.data_root)?; + let overlaps = source_files + .keys() + .filter(|path| target_files.contains_key(*path)) + .cloned() + .collect::>(); + let mut differing = Vec::new(); + for path in &overlaps { + if is_runtime_lock(path) || is_sqlite_database(path) || is_sqlite_sidecar(path) { + continue; + } + if file_digest(&source.data_root.join(path))? != file_digest(&target.data_root.join(path))? + { + differing.push(path.clone()); + } + } + let db = sqlite::inspect_collisions( + &evidence.sessions, + &source.sessions_db_path, + &target.sessions_db_path, + ) + .await?; + Ok(CollisionSummary { + fact_content_overlaps: evidence + .source_graph + .identities + .fact_overlap(&evidence.target_graph.identities), + session_overlaps: db.sessions, + message_overlaps: db.messages, + lcm_message_overlaps: db.lcm_messages, + artifact_path_overlaps: overlaps.len(), + differing_artifact_paths: differing, + semantics: vec![ + "facts: union by content; tags/entities/metadata are merged, counters take maxima, newest trust/category wins, feedback events are deduplicated".to_string(), + "sessions: union by provider/session id; time bounds widen and non-null target fields win".to_string(), + "messages and LCM payload identities: identical rows deduplicate; divergent content is a hard error".to_string(), + "branch graphs: target branches retain their names; every source branch is preserved under consolidated//...".to_string(), + "artifact paths: identical files deduplicate; divergent non-reference files are preserved under consolidation-preserved; divergent payload/handle files are a hard error".to_string(), + ], + }) +} + +fn destination_project_id(git_common_dir: &Path, source: &str, target: &str) -> String { + let mut ids = [source, target]; + ids.sort_unstable(); + let mut hash = Sha256::new(); + hash.update(b"tracedecay-profile-consolidation-v1\0"); + hash.update( + canonical_or_original(git_common_dir) + .to_string_lossy() + .as_bytes(), + ); + hash.update(b"\0"); + hash.update(ids[0].as_bytes()); + hash.update(b"\0"); + hash.update(ids[1].as_bytes()); + format!("proj_{}", &hex::encode(hash.finalize())[..16]) +} + +fn confirmation_token(fingerprint: &str, migration_id: &str) -> String { + let mut hash = Sha256::new(); + hash.update(b"tracedecay-consolidation-confirm-v1\0"); + hash.update(migration_id.as_bytes()); + hash.update(b"\0"); + hash.update(fingerprint.as_bytes()); + format!("confirm-{}", &hex::encode(hash.finalize())[..24]) +} + +fn fingerprint_inputs( + evidence: &InputReadEvidence, + source: &StoreLayout, + target: &StoreLayout, +) -> Result { + let mut hash = Sha256::new(); + for (label, root, graph) in [ + ("source", &source.data_root, &evidence.source_graph), + ("target", &target.data_root, &evidence.target_graph), + ] { + hash.update(label.as_bytes()); + for (relative, path) in relative_file_map(root)? { + if is_runtime_lock(&relative) || is_sqlite_sidecar(&relative) { + continue; + } + hash.update(relative.to_string_lossy().as_bytes()); + if is_sqlite_database(&relative) { + let fingerprint = graph + .fingerprints + .get(&path) + .or_else(|| evidence.session_fingerprints.get(&path)) + .ok_or_else(|| { + config_error(format!( + "missing logical fingerprint for '{}'", + path.display() + )) + })?; + hash.update(fingerprint); + } else { + let fingerprint = file_digest(&path)?; + hash.update(fingerprint); + } + } + } + Ok(hex::encode(hash.finalize())) +} + +fn load_or_create_ledger(resolved: &ResolvedPlan, path: &Path) -> Result { + if let Some(ledger) = load_ledger(path)? { + return Ok(ledger); + } + if resolved.report.destination_data_root.exists() { + return Err(config_error(format!( + "destination shard '{}' already exists without this migration ledger", + resolved.report.destination_data_root.display() + ))); + } + let ledger = ConsolidationLedger { + schema_version: LEDGER_SCHEMA_VERSION, + migration_id: resolved.report.migration_id.clone(), + confirmation_token: resolved.report.confirmation_token.clone(), + input_fingerprint: resolved.input_fingerprint.clone(), + source_project_id: resolved.report.source.project_id.clone(), + target_project_id: resolved.report.target.project_id.clone(), + destination_project_id: resolved.report.destination_project_id.clone(), + project_root: resolved.report.project_root.clone(), + git_common_dir: resolved.report.git_common_dir.clone(), + state: ConsolidationState::Planned, + graph_offsets: Vec::new(), + session_offsets: None, + preserved_collisions: Vec::new(), + }; + save_ledger(path, &ledger)?; + Ok(ledger) +} + +fn validate_ledger(ledger: &ConsolidationLedger, resolved: &ResolvedPlan) -> Result<()> { + if ledger.schema_version != LEDGER_SCHEMA_VERSION + || ledger.migration_id != resolved.report.migration_id + || ledger.confirmation_token != resolved.report.confirmation_token + || ledger.input_fingerprint != resolved.input_fingerprint + || ledger.source_project_id != resolved.report.source.project_id + || ledger.target_project_id != resolved.report.target.project_id + || ledger.destination_project_id != resolved.report.destination_project_id + || !same_path(&ledger.git_common_dir, &resolved.report.git_common_dir) + { + return Err(config_error( + "existing consolidation ledger does not match the current immutable input inventory", + )); + } + Ok(()) +} + +fn load_ledger(path: &Path) -> Result> { + let bytes = match fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(io_error(error)), + }; + serde_json::from_slice(&bytes).map(Some).map_err(|error| { + config_error(format!( + "consolidation ledger '{}' is corrupt: {error}", + path.display() + )) + }) +} + +fn save_ledger(path: &Path, ledger: &ConsolidationLedger) -> Result<()> { + let bytes = + serde_json::to_vec_pretty(ledger).map_err(|error| config_error(error.to_string()))?; + let temp = path.with_extension(format!("json.tmp-{}", std::process::id())); + PrivateStoreIo::write_file_atomically(path, &temp, &bytes).map_err(io_error) +} + +fn backup_store(layout: &StoreLayout, backup_root: &Path) -> Result<()> { + let project_id = layout.identity.project_id.as_deref().unwrap_or("unknown"); + copy_tree_exact(&layout.data_root, &backup_root.join(project_id)) +} + +async fn merge_databases(resolved: &ResolvedPlan, ledger: &mut ConsolidationLedger) -> Result<()> { + let destination = &resolved.report.destination_data_root; + let meta = load_required_branch_meta(&layout_for_id( + &resolved.report.project_root, + destination + .parent() + .and_then(Path::parent) + .unwrap_or(Path::new("")), + &resolved.report.destination_project_id, + )?)?; + let graph_paths = graph_db_paths_for_root(destination, &meta)?; + if ledger.graph_offsets.is_empty() { + ledger.graph_offsets = sqlite::plan_graph_offsets(&graph_paths).await?; + save_ledger(&resolved.report.ledger_path, ledger)?; + } + sqlite::merge_graph_facts(&graph_paths, &ledger.graph_offsets).await?; + + let input_root = destination.join(INPUT_DIR); + fs::create_dir_all(&input_root).map_err(io_error)?; + let source_sessions = input_root.join("source-sessions.db"); + if !source_sessions.is_file() { + copy_sqlite_family_exact(&resolved.source_layout.sessions_db_path, &source_sessions)?; + } + let target_sessions = destination.join(storage::SESSIONS_DB_FILENAME); + if ledger.session_offsets.is_none() { + ledger.session_offsets = + Some(sqlite::plan_session_offsets(&target_sessions, &source_sessions).await?); + save_ledger(&resolved.report.ledger_path, ledger)?; + } + let target_input = input_root.join("target-sessions.db"); + if !target_input.is_file() { + copy_sqlite_family_exact(&target_sessions, &target_input)?; + } + let offsets = ledger + .session_offsets + .as_ref() + .ok_or_else(|| config_error("session merge offsets are missing from the ledger"))?; + sqlite::merge_sessions(&target_sessions, &source_sessions, offsets).await?; + Ok(()) +} + +fn merge_non_database_artifacts( + resolved: &ResolvedPlan, + ledger: &mut ConsolidationLedger, +) -> Result<()> { + let source = &resolved.source_layout.data_root; + let destination = &resolved.report.destination_data_root; + for (relative, path) in relative_file_map(source)? { + if excluded_source_artifact(&relative) { + continue; + } + let target = destination.join(&relative); + if !target.exists() { + copy_file_atomic(&path, &target)?; + continue; + } + if file_digest(&path)? == file_digest(&target)? { + continue; + } + if is_reference_artifact(&relative) { + return Err(config_error(format!( + "divergent referenced artifact collision at '{}'; both inputs and backups remain unchanged", + relative.display() + ))); + } + let preserved = destination + .join(PRESERVED_DIR) + .join(&resolved.report.source.project_id) + .join(&relative); + copy_file_atomic(&path, &preserved)?; + ledger.preserved_collisions.push(relative); + } + ledger.preserved_collisions.sort(); + ledger.preserved_collisions.dedup(); + Ok(()) +} + +fn remove_verification_inputs(resolved: &ResolvedPlan) -> Result<()> { + let input = resolved.report.destination_data_root.join(INPUT_DIR); + match fs::remove_dir_all(input) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(io_error(error)), + } +} + +fn write_destination_manifest(resolved: &ResolvedPlan) -> Result<()> { + let layout = layout_for_id( + &resolved.report.project_root, + resolved + .report + .destination_data_root + .parent() + .and_then(Path::parent) + .ok_or_else(|| config_error("destination shard has no profile root"))?, + &resolved.report.destination_project_id, + )?; + storage::write_store_manifest(&layout).map(|_| ()) +} + +fn graph_db_paths(layout: &StoreLayout, meta: &BranchMeta) -> Result> { + graph_db_paths_for_root(&layout.data_root, meta) +} + +fn input_database_paths(resolved: &ResolvedPlan) -> Result> { + database_paths_for_layouts( + &resolved.source_layout, + &resolved.source_meta, + &resolved.target_layout, + &resolved.target_meta, + ) +} + +fn database_paths_for_layouts( + source_layout: &StoreLayout, + source_meta: &BranchMeta, + target_layout: &StoreLayout, + target_meta: &BranchMeta, +) -> Result> { + let mut paths = graph_db_paths(source_layout, source_meta)?; + paths.extend(graph_db_paths(target_layout, target_meta)?); + paths.push(source_layout.sessions_db_path.clone()); + paths.push(target_layout.sessions_db_path.clone()); + paths.sort(); + paths.dedup(); + Ok(paths) +} + +fn graph_db_paths_for_root(root: &Path, meta: &BranchMeta) -> Result> { + let main = root.join(crate::config::DB_FILENAME); + let mut paths = BTreeSet::new(); + for entry in meta.branches.values() { + let path = root.join(&entry.db_file); + if !path.is_file() { + return Err(config_error(format!( + "branch graph '{}' is missing", + path.display() + ))); + } + paths.insert(path); + } + if !paths.remove(&main) { + return Err(config_error(format!( + "default graph '{}' is not present in branch metadata", + main.display() + ))); + } + Ok(std::iter::once(main).chain(paths).collect()) +} + +fn same_path(left: &Path, right: &Path) -> bool { + canonical_or_original(left) == canonical_or_original(right) +} + +fn canonical_or_original(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +fn migration_scratch_root(profile_root: &Path) -> Result { + let parent = profile_root + .parent() + .ok_or_else(|| config_error("profile root has no parent for private migration scratch"))?; + let mut hash = Sha256::new(); + hash.update(profile_root.to_string_lossy().as_bytes()); + let prefix = format!( + ".tracedecay-migration-scratch-{}-{}", + &hex::encode(hash.finalize())[..12], + std::process::id() + ); + for _ in 0..100 { + let sequence = NEXT_MIGRATION_SCRATCH.fetch_add(1, Ordering::Relaxed); + let path = parent.join(format!("{prefix}-{sequence}")); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(&path) { + Ok(()) => return Ok(MigrationScratchRoot { path }), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(io_error(error)), + } + } + Err(config_error( + "could not allocate a private migration scratch directory", + )) +} + +fn git_remote_url(project_root: &Path) -> Option { + let repo = gix::discover(project_root).ok()?; + let value = repo.config_snapshot().string("remote.origin.url")?; + let value = value.to_string(); + let value = value.trim(); + (!value.is_empty()).then(|| value.to_string()) +} + +fn config_error(message: impl Into) -> TraceDecayError { + TraceDecayError::Config { + message: message.into(), + } +} + +#[allow(clippy::needless_pass_by_value)] +fn io_error(error: io::Error) -> TraceDecayError { + config_error(error.to_string()) +} + +#[cfg(test)] +mod tests; diff --git a/src/migrate/consolidate/preflight.rs b/src/migrate/consolidate/preflight.rs new file mode 100644 index 00000000..6b170104 --- /dev/null +++ b/src/migrate/consolidate/preflight.rs @@ -0,0 +1,204 @@ +use std::fs::{self, File, OpenOptions}; + +use fs2::FileExt; + +use super::*; + +pub(super) struct StoreLocks { + files: Vec, +} + +impl Drop for StoreLocks { + fn drop(&mut self) { + for file in &self.files { + let _ = FileExt::unlock(file); + } + } +} + +pub(super) fn ensure_profile_offline(options: &ConsolidationOptions) -> Result<()> { + if crate::config::user_data_dir().is_some_and(|root| same_path(&root, &options.profile_root)) + && crate::daemon::daemon_reachable() + { + return Err(config_error( + "profile shard consolidation is offline-only, including its dry-run; stop the TraceDecay daemon and all MCP/CLI writers, then retry", + )); + } + Ok(()) +} + +pub(super) fn ensure_no_open_store_holders(database_paths: &[PathBuf]) -> Result<()> { + evaluate_holder_scan(crate::open_store_holders::scan(database_paths).map_err(io_error)?) +} + +fn evaluate_holder_scan(scan: crate::open_store_holders::OpenStoreHolderScan) -> Result<()> { + match scan { + crate::open_store_holders::OpenStoreHolderScan::Supported(holders) + if holders.is_empty() => + { + Ok(()) + } + crate::open_store_holders::OpenStoreHolderScan::Supported(holders) => { + let mut details = String::new(); + for holder in holders { + let version = holder.version.as_deref().unwrap_or("version unknown"); + let executable = holder.executable.as_deref().map_or_else( + || "executable unknown".to_string(), + |path| path.display().to_string(), + ); + let paths = holder + .paths + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", "); + details.push_str(&format!( + "\n- pid {}: {} [{}; {}]; open: {}", + holder.pid, + truncate_command(&holder.command), + version, + executable, + paths + )); + } + Err(config_error(format!( + "profile shard consolidation requires every input store handle to be closed; restart the listed agent hosts and retry (TraceDecay never terminates them automatically):{details}" + ))) + } + crate::open_store_holders::OpenStoreHolderScan::Unsupported { reason } => { + Err(config_error(format!( + "profile shard consolidation cannot prove every input store handle is closed: {reason}; run consolidation on a host with open-store process discovery" + ))) + } + } +} + +fn truncate_command(command: &str) -> String { + const LIMIT: usize = 200; + let mut chars = command.chars(); + let prefix = chars.by_ref().take(LIMIT).collect::(); + if chars.next().is_some() { + format!("{prefix}...") + } else { + prefix + } +} + +pub(super) fn acquire_store_locks( + source: &StoreLayout, + target: &StoreLayout, +) -> Result { + let mut paths = vec![ + source.sync_lock_path.clone(), + source.branch_add_lock_path.clone(), + target.sync_lock_path.clone(), + target.branch_add_lock_path.clone(), + ]; + paths.sort(); + paths.dedup(); + let mut files = Vec::new(); + for path in paths { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(io_error)?; + } + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options.open(&path).map_err(io_error)?; + file.try_lock_exclusive().map_err(|error| { + config_error(format!( + "store is busy at '{}': {error}; stop MCP/CLI writers and retry", + path.display() + )) + })?; + files.push(file); + } + Ok(StoreLocks { files }) +} + +pub(super) fn preflight_disk_space(resolved: &ResolvedPlan) -> Result<()> { + const LEDGER_HEADROOM: u64 = 1024 * 1024; + let source_backup = resolved + .report + .backup_root + .join(&resolved.report.source.project_id); + let target_backup = resolved + .report + .backup_root + .join(&resolved.report.target.project_id); + let backup_bytes = + additional_copy_bytes(&resolved.source_layout.data_root, &source_backup)?.saturating_add( + additional_copy_bytes(&resolved.target_layout.data_root, &target_backup)?, + ); + let destination_ceiling = resolved + .report + .source + .bytes + .saturating_add(resolved.report.target.bytes); + let existing_destination = if resolved.report.destination_data_root.is_dir() { + tree_stats(&resolved.report.destination_data_root)?.1 + } else { + 0 + }; + let destination_bytes = destination_ceiling.saturating_sub(existing_destination); + let required = backup_bytes + .saturating_add(destination_bytes) + .saturating_add(LEDGER_HEADROOM); + let profile_root = resolved + .report + .destination_data_root + .parent() + .and_then(Path::parent) + .ok_or_else(|| config_error("destination shard has no profile root"))?; + let available = fs2::available_space(profile_root).map_err(io_error)?; + if required > available { + return Err(config_error(format!( + "insufficient profile space before consolidation: required {required} bytes \ + (backups {backup_bytes}, destination upper bound {destination_bytes}, ledger {LEDGER_HEADROOM}; \ + scratch fallback already reserved {}), available {available} at '{}'; no ledger or backup was changed", + resolved.evidence.sessions.copied_bytes(), + profile_root.display() + ))); + } + Ok(()) +} + +fn additional_copy_bytes(source: &Path, target: &Path) -> Result { + let mut bytes = 0_u64; + for (relative, path) in relative_file_map(source)? { + if !target.join(relative).exists() { + bytes = bytes.saturating_add(fs::metadata(path).map_err(io_error)?.len()); + } + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::{evaluate_holder_scan, truncate_command}; + use crate::open_store_holders::OpenStoreHolderScan; + + #[test] + fn holder_command_is_bounded_on_character_boundaries() { + let command = "x".repeat(205); + assert_eq!(truncate_command(&command).len(), 203); + assert!(truncate_command("trace decay").ends_with("decay")); + } + + #[test] + fn unsupported_holder_discovery_never_silently_weakens_offline_safety() { + let error = evaluate_holder_scan(OpenStoreHolderScan::Unsupported { + reason: "synthetic unsupported host".to_string(), + }) + .unwrap_err(); + assert!(error.to_string().contains("cannot prove"), "{error}"); + assert!( + error.to_string().contains("synthetic unsupported host"), + "{error}" + ); + } +} diff --git a/src/migrate/consolidate/prepare.rs b/src/migrate/consolidate/prepare.rs new file mode 100644 index 00000000..7c2682aa --- /dev/null +++ b/src/migrate/consolidate/prepare.rs @@ -0,0 +1,244 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +use super::files::{remove_runtime_files, sqlite_sidecar}; +use super::*; +use crate::branch_meta::BranchEntry; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PrepareStop { + TargetCopy, + SourceBranch(usize), + BranchMetaWrite, + Publish, +} + +pub(super) fn prepare_destination(resolved: &ResolvedPlan) -> Result<()> { + prepare_destination_with_stop(resolved, None) +} + +pub(super) fn prepare_destination_with_stop( + resolved: &ResolvedPlan, + stop: Option, +) -> Result<()> { + let destination = &resolved.report.destination_data_root; + if destination.exists() { + validate_prepared_root(resolved, destination)?; + return Ok(()); + } + + let staging = staging_root(resolved)?; + create_private_directory(&staging)?; + copy_target_tree(resolved, &staging)?; + remove_runtime_files(&staging)?; + maybe_stop(stop, PrepareStop::TargetCopy)?; + + let mut merged_meta = resolved.target_meta.clone(); + preserve_source_branch_graphs(resolved, &staging, &mut merged_meta, stop)?; + branch_meta::save_branch_meta(&staging, &merged_meta).map_err(io_error)?; + maybe_stop(stop, PrepareStop::BranchMetaWrite)?; + validate_prepared_root(resolved, &staging)?; + sync_directory(&staging)?; + + fs::rename(&staging, destination).map_err(io_error)?; + files::sync_parent_directory( + destination + .parent() + .ok_or_else(|| config_error("destination shard has no parent"))?, + )?; + maybe_stop(stop, PrepareStop::Publish)?; + Ok(()) +} + +fn copy_target_tree(resolved: &ResolvedPlan, staging: &Path) -> Result<()> { + for (relative, source) in relative_file_map(&resolved.target_layout.data_root)? { + if relative == Path::new(storage::BRANCH_META_FILENAME) || is_runtime_lock(&relative) { + continue; + } + files::copy_file_exact(&source, &staging.join(relative))?; + } + Ok(()) +} + +fn preserve_source_branch_graphs( + resolved: &ResolvedPlan, + staging: &Path, + merged: &mut BranchMeta, + stop: Option, +) -> Result<()> { + let prefix = format!("consolidated/{}/", resolved.report.source.project_id); + let mut branches = resolved.source_meta.branches.iter().collect::>(); + branches.sort_by_key(|(name, _)| *name); + for (index, (branch_name, entry)) in branches.into_iter().enumerate() { + let source_db = resolved.source_layout.data_root.join(&entry.db_file); + let merged_name = format!("{prefix}{branch_name}"); + let stem = source_branch_stem(&merged_name); + if stem.is_empty() { + return Err(config_error(format!( + "source branch '{branch_name}' cannot be represented safely" + ))); + } + let relative = PathBuf::from("branches").join(format!("{stem}.db")); + copy_sqlite_family_exact(&source_db, &staging.join(&relative))?; + merged.branches.insert( + merged_name, + BranchEntry { + db_file: relative.to_string_lossy().to_string(), + parent: entry + .parent + .as_deref() + .map(|parent| format!("{prefix}{parent}")), + created_at: entry.created_at.clone(), + last_synced_at: entry.last_synced_at.clone(), + }, + ); + maybe_stop(stop, PrepareStop::SourceBranch(index + 1))?; + } + Ok(()) +} + +fn validate_prepared_root(resolved: &ResolvedPlan, root: &Path) -> Result<()> { + let (expected_meta, expected_files) = expected_prepared_files(resolved)?; + let actual_meta = branch_meta::load_branch_meta(root) + .ok_or_else(|| config_error("prepared destination branch metadata is invalid"))?; + if serde_json::to_value(&actual_meta).map_err(|error| config_error(error.to_string()))? + != serde_json::to_value(&expected_meta).map_err(|error| config_error(error.to_string()))? + { + return Err(config_error( + "prepared destination branch metadata differs from the deterministic plan", + )); + } + + let actual = relative_file_map(root)?; + let actual_paths = actual.keys().cloned().collect::>(); + let mut expected_paths = expected_files.keys().cloned().collect::>(); + expected_paths.insert(PathBuf::from(storage::BRANCH_META_FILENAME)); + if actual_paths != expected_paths { + return Err(config_error(format!( + "prepared destination file inventory differs from the deterministic plan: expected {expected_paths:?}, found {actual_paths:?}" + ))); + } + for (relative, source) in expected_files { + if file_digest(&source)? != file_digest(&root.join(&relative))? { + return Err(config_error(format!( + "prepared destination artifact '{}' differs from '{}'", + root.join(relative).display(), + source.display() + ))); + } + } + Ok(()) +} + +fn expected_prepared_files( + resolved: &ResolvedPlan, +) -> Result<(BranchMeta, BTreeMap)> { + let mut files = BTreeMap::new(); + for (relative, source) in relative_file_map(&resolved.target_layout.data_root)? { + if relative == Path::new(storage::BRANCH_META_FILENAME) || is_runtime_lock(&relative) { + continue; + } + files.insert(relative, source); + } + + let prefix = format!("consolidated/{}/", resolved.report.source.project_id); + let mut meta = resolved.target_meta.clone(); + let mut branches = resolved.source_meta.branches.iter().collect::>(); + branches.sort_by_key(|(name, _)| *name); + for (branch_name, entry) in branches { + let source_db = resolved.source_layout.data_root.join(&entry.db_file); + let merged_name = format!("{prefix}{branch_name}"); + let relative = + PathBuf::from("branches").join(format!("{}.db", source_branch_stem(&merged_name))); + for (suffix, source) in [ + ("", source_db.clone()), + ("-wal", sqlite_sidecar(&source_db, "-wal")), + ("-shm", sqlite_sidecar(&source_db, "-shm")), + ] { + if source.is_file() { + let target = if suffix.is_empty() { + relative.clone() + } else { + sqlite_sidecar(&relative, suffix) + }; + files.insert(target, source); + } + } + meta.branches.insert( + merged_name, + BranchEntry { + db_file: relative.to_string_lossy().to_string(), + parent: entry + .parent + .as_deref() + .map(|parent| format!("{prefix}{parent}")), + created_at: entry.created_at.clone(), + last_synced_at: entry.last_synced_at.clone(), + }, + ); + } + Ok((meta, files)) +} + +fn staging_root(resolved: &ResolvedPlan) -> Result { + let parent = resolved + .report + .destination_data_root + .parent() + .ok_or_else(|| config_error("destination shard has no parent"))?; + Ok(parent.join(format!(".{}.staging", resolved.report.migration_id))) +} + +fn source_branch_stem(branch_name: &str) -> String { + let base = crate::branch::sanitize_branch_name(branch_name) + .chars() + .take(180) + .collect::(); + if base.is_empty() { + return base; + } + let mut hash = Sha256::new(); + hash.update(branch_name.as_bytes()); + format!("{base}_{}", &hex::encode(hash.finalize())[..10]) +} + +fn maybe_stop(stop: Option, point: PrepareStop) -> Result<()> { + if stop == Some(point) { + Err(config_error(format!( + "synthetic interruption inside destination preparation at {point:?}" + ))) + } else { + Ok(()) + } +} + +fn create_private_directory(path: &Path) -> Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => return Ok(()), + Ok(_) => return Err(config_error("destination staging path is not a directory")), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(io_error(error)), + } + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(path).map_err(io_error) +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> Result<()> { + std::fs::File::open(path) + .and_then(|directory| directory.sync_all()) + .map_err(io_error) +} + +#[cfg(not(unix))] +fn sync_directory(_path: &Path) -> Result<()> { + Ok(()) +} diff --git a/src/migrate/consolidate/sqlite.rs b/src/migrate/consolidate/sqlite.rs new file mode 100644 index 00000000..87b77f40 --- /dev/null +++ b/src/migrate/consolidate/sqlite.rs @@ -0,0 +1,792 @@ +use std::path::{Path, PathBuf}; + +use libsql::{Connection, params}; +use serde::{Deserialize, Serialize}; + +use crate::db::Database; +use crate::errors::{Result, TraceDecayError}; +use crate::global_db::GlobalDb; +use crate::memory::store::MemoryStore; + +mod inspect; +mod verify; + +#[cfg(test)] +pub(super) use inspect::count_rows; +pub(super) use inspect::{ + GraphLogicalIdentities, acquire_offline_guards, count_rows_in, extend_graph_identities, + inspect_collisions, quick_check_connection, quick_check_in, +}; +pub(super) use verify::verify_session_union_sql; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(super) struct GraphMergeOffsets { + pub source_path: PathBuf, + pub fact_id: i64, + pub entity_id: i64, + pub feedback_id: i64, + pub oplog_id: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(super) struct SessionMergeOffsets { + pub raw: i64, + pub span: i64, + pub savings: i64, + pub analytics: i64, +} + +pub(super) async fn plan_graph_offsets(paths: &[PathBuf]) -> Result> { + let (target_path, source_paths) = paths + .split_first() + .ok_or_else(|| db_message("plan_graph_offsets", "no graph databases were supplied"))?; + normalize_graph(target_path).await?; + for path in source_paths { + normalize_graph(path).await?; + } + let mut maxima = graph_maxima(target_path).await?; + let mut offsets = Vec::new(); + for path in source_paths { + let source = graph_maxima(path).await?; + let offset = GraphMergeOffsets { + source_path: path.clone(), + fact_id: maxima.0, + entity_id: maxima.1, + feedback_id: maxima.2, + oplog_id: maxima.3, + }; + maxima.0 = checked_advance(maxima.0, source.0, "fact_id")?; + maxima.1 = checked_advance(maxima.1, source.1, "entity_id")?; + maxima.2 = checked_advance(maxima.2, source.2, "feedback_id")?; + maxima.3 = checked_advance(maxima.3, source.3, "oplog_id")?; + offsets.push(offset); + } + Ok(offsets) +} + +pub(super) async fn merge_graph_facts( + paths: &[PathBuf], + offsets: &[GraphMergeOffsets], +) -> Result<()> { + let target_path = paths + .first() + .ok_or_else(|| db_message("merge_graph_facts", "no target graph database"))?; + let (target, _) = Database::open(target_path).await?; + for offset in offsets { + merge_one_graph(target.conn(), offset).await?; + } + MemoryStore::new(target.conn()).rebuild_all_banks().await?; + target.checkpoint().await?; + target.close(); + Ok(()) +} + +async fn normalize_graph(path: &Path) -> Result<()> { + let (db, _) = Database::open(path).await?; + db.checkpoint().await?; + db.close(); + Ok(()) +} + +async fn graph_maxima(path: &Path) -> Result<(i64, i64, i64, i64)> { + let (db, _) = Database::open_read_only(path).await?; + let result = ( + table_max(db.conn(), "memory_facts", "fact_id").await?, + table_max(db.conn(), "memory_entities", "entity_id").await?, + table_max(db.conn(), "memory_feedback_events", "event_id").await?, + table_max(db.conn(), "memory_oplog", "id").await?, + ); + db.close(); + Ok(result) +} + +async fn merge_one_graph(conn: &Connection, offset: &GraphMergeOffsets) -> Result<()> { + attach_as(conn, &offset.source_path, "source").await?; + conn.execute("PRAGMA foreign_keys = OFF", ()) + .await + .map_err(|error| db_error("merge_graph_facts", error))?; + conn.execute("BEGIN IMMEDIATE", ()) + .await + .map_err(|error| db_error("merge_graph_facts", error))?; + let result = merge_one_graph_tx(conn, offset).await; + match result { + Ok(()) => conn + .execute("COMMIT", ()) + .await + .map_err(|error| db_error("merge_graph_facts", error))?, + Err(error) => { + let _ = conn.execute("ROLLBACK", ()).await; + let _ = conn.execute("DETACH DATABASE source", ()).await; + return Err(error); + } + }; + conn.execute("DETACH DATABASE source", ()) + .await + .map_err(|error| db_error("merge_graph_facts", error))?; + Ok(()) +} + +async fn merge_one_graph_tx(conn: &Connection, offset: &GraphMergeOffsets) -> Result<()> { + conn.execute_batch(&format!( + "CREATE TEMP TABLE IF NOT EXISTS consolidation_fact_map( + source_id INTEGER PRIMARY KEY, target_id INTEGER NOT NULL + ); + DELETE FROM consolidation_fact_map; + CREATE TEMP TABLE IF NOT EXISTS consolidation_entity_map( + source_id INTEGER PRIMARY KEY, target_id INTEGER NOT NULL + ); + DELETE FROM consolidation_entity_map; + + INSERT OR IGNORE INTO memory_facts ( + fact_id, content, category, tags, trust_score, retrieval_count, + access_count, helpful_count, unhelpful_count, created_at, updated_at, + last_retrieved_at, last_recalled_at, last_feedback_at, source, + metadata, hrr_vector, hrr_algebra, hrr_dim, hrr_precision + ) + SELECT fact_id + {fact}, content, category, tags, trust_score, + retrieval_count, access_count, helpful_count, unhelpful_count, + created_at, updated_at, last_retrieved_at, last_recalled_at, + last_feedback_at, source, metadata, hrr_vector, hrr_algebra, + hrr_dim, hrr_precision + FROM source.memory_facts s + WHERE NOT EXISTS ( + SELECT 1 FROM memory_facts t WHERE t.content = s.content + ); + + UPDATE memory_facts AS t SET + tags = (SELECT json_group_array(value) FROM ( + SELECT value FROM json_each(t.tags) + UNION + SELECT value FROM json_each(( + SELECT s.tags FROM source.memory_facts s WHERE s.content = t.content + )) ORDER BY value + )), + category = CASE WHEN COALESCE(( + SELECT s.updated_at FROM source.memory_facts s WHERE s.content = t.content + ), -1) > t.updated_at THEN ( + SELECT s.category FROM source.memory_facts s WHERE s.content = t.content + ) ELSE t.category END, + trust_score = CASE WHEN COALESCE(( + SELECT s.last_feedback_at FROM source.memory_facts s WHERE s.content = t.content + ), -1) > COALESCE(t.last_feedback_at, -1) THEN ( + SELECT s.trust_score FROM source.memory_facts s WHERE s.content = t.content + ) ELSE t.trust_score END, + retrieval_count = MAX(t.retrieval_count, COALESCE(( + SELECT s.retrieval_count FROM source.memory_facts s WHERE s.content = t.content + ), 0)), + access_count = MAX(t.access_count, COALESCE(( + SELECT s.access_count FROM source.memory_facts s WHERE s.content = t.content + ), 0)), + helpful_count = MAX(t.helpful_count, COALESCE(( + SELECT s.helpful_count FROM source.memory_facts s WHERE s.content = t.content + ), 0)), + unhelpful_count = MAX(t.unhelpful_count, COALESCE(( + SELECT s.unhelpful_count FROM source.memory_facts s WHERE s.content = t.content + ), 0)), + created_at = MIN(t.created_at, COALESCE(( + SELECT s.created_at FROM source.memory_facts s WHERE s.content = t.content + ), t.created_at)), + updated_at = MAX(t.updated_at, COALESCE(( + SELECT s.updated_at FROM source.memory_facts s WHERE s.content = t.content + ), t.updated_at)), + last_retrieved_at = CASE + WHEN t.last_retrieved_at IS NULL THEN (SELECT s.last_retrieved_at FROM source.memory_facts s WHERE s.content = t.content) + WHEN (SELECT s.last_retrieved_at FROM source.memory_facts s WHERE s.content = t.content) IS NULL THEN t.last_retrieved_at + ELSE MAX(t.last_retrieved_at, (SELECT s.last_retrieved_at FROM source.memory_facts s WHERE s.content = t.content)) END, + last_recalled_at = CASE + WHEN t.last_recalled_at IS NULL THEN (SELECT s.last_recalled_at FROM source.memory_facts s WHERE s.content = t.content) + WHEN (SELECT s.last_recalled_at FROM source.memory_facts s WHERE s.content = t.content) IS NULL THEN t.last_recalled_at + ELSE MAX(t.last_recalled_at, (SELECT s.last_recalled_at FROM source.memory_facts s WHERE s.content = t.content)) END, + last_feedback_at = CASE + WHEN t.last_feedback_at IS NULL THEN (SELECT s.last_feedback_at FROM source.memory_facts s WHERE s.content = t.content) + WHEN (SELECT s.last_feedback_at FROM source.memory_facts s WHERE s.content = t.content) IS NULL THEN t.last_feedback_at + ELSE MAX(t.last_feedback_at, (SELECT s.last_feedback_at FROM source.memory_facts s WHERE s.content = t.content)) END, + metadata = json_patch(COALESCE(( + SELECT s.metadata FROM source.memory_facts s WHERE s.content = t.content + ), '{{}}'), t.metadata) + WHERE EXISTS (SELECT 1 FROM source.memory_facts s WHERE s.content = t.content); + + INSERT INTO consolidation_fact_map(source_id, target_id) + SELECT s.fact_id, t.fact_id + FROM source.memory_facts s JOIN memory_facts t ON t.content = s.content; + + INSERT OR IGNORE INTO memory_entities ( + entity_id, name, normalized_name, entity_type, aliases, created_at + ) + SELECT entity_id + {entity}, name, normalized_name, entity_type, aliases, created_at + FROM source.memory_entities s + WHERE NOT EXISTS ( + SELECT 1 FROM memory_entities t WHERE t.normalized_name = s.normalized_name + ); + + UPDATE memory_entities AS t SET + aliases = (SELECT json_group_array(value) FROM ( + SELECT value FROM json_each(t.aliases) + UNION + SELECT value FROM json_each(( + SELECT s.aliases FROM source.memory_entities s + WHERE s.normalized_name = t.normalized_name + )) ORDER BY value + )), + created_at = MIN(t.created_at, COALESCE(( + SELECT s.created_at FROM source.memory_entities s + WHERE s.normalized_name = t.normalized_name + ), t.created_at)) + WHERE EXISTS ( + SELECT 1 FROM source.memory_entities s + WHERE s.normalized_name = t.normalized_name + ); + + INSERT INTO consolidation_entity_map(source_id, target_id) + SELECT s.entity_id, t.entity_id + FROM source.memory_entities s + JOIN memory_entities t ON t.normalized_name = s.normalized_name; + + INSERT OR IGNORE INTO memory_fact_entities(fact_id, entity_id) + SELECT fm.target_id, em.target_id + FROM source.memory_fact_entities sfe + JOIN consolidation_fact_map fm ON fm.source_id = sfe.fact_id + JOIN consolidation_entity_map em ON em.source_id = sfe.entity_id; + + INSERT OR IGNORE INTO memory_feedback_events ( + event_id, fact_id, action, trust_delta, old_trust, new_trust, + created_at, source, note + ) + SELECT e.event_id + {feedback}, fm.target_id, e.action, e.trust_delta, + e.old_trust, e.new_trust, e.created_at, e.source, e.note + FROM source.memory_feedback_events e + JOIN consolidation_fact_map fm ON fm.source_id = e.fact_id + WHERE NOT EXISTS ( + SELECT 1 FROM memory_feedback_events t + WHERE t.fact_id = fm.target_id + AND t.action = e.action + AND t.trust_delta = e.trust_delta + AND t.old_trust = e.old_trust + AND t.new_trust = e.new_trust + AND t.created_at = e.created_at + AND t.source = e.source + AND t.note IS e.note + ); + + UPDATE memory_facts AS f SET + helpful_count = MAX(f.helpful_count, ( + SELECT COUNT(*) FROM memory_feedback_events e + WHERE e.fact_id=f.fact_id AND e.action='helpful' + )), + unhelpful_count = MAX(f.unhelpful_count, ( + SELECT COUNT(*) FROM memory_feedback_events e + WHERE e.fact_id=f.fact_id AND e.action='unhelpful' + )) + WHERE f.fact_id IN (SELECT target_id FROM consolidation_fact_map); + + INSERT OR IGNORE INTO memory_oplog(id, ts, op, fact_id, detail_json) + SELECT o.id + {oplog}, o.ts, o.op, fm.target_id, o.detail_json + FROM source.memory_oplog o + LEFT JOIN consolidation_fact_map fm ON fm.source_id = o.fact_id;", + fact = offset.fact_id, + entity = offset.entity_id, + feedback = offset.feedback_id, + oplog = offset.oplog_id, + )) + .await + .map_err(|error| db_error("merge_graph_facts", error))?; + Ok(()) +} + +pub(super) async fn plan_session_offsets( + target: &Path, + source: &Path, +) -> Result { + normalize_sessions(target).await?; + normalize_sessions(source).await?; + reject_session_registry_rows(source).await?; + Ok(SessionMergeOffsets { + raw: db_table_max(target, "lcm_raw_messages", "store_id").await?, + span: db_table_max(target, "session_git_spans", "span_id").await?, + savings: db_table_max(target, "savings_ledger", "id").await?, + analytics: db_table_max(target, "analytics_events", "id").await?, + }) +} + +pub(super) async fn merge_sessions( + target_path: &Path, + source_path: &Path, + offsets: &SessionMergeOffsets, +) -> Result<()> { + normalize_sessions(target_path).await?; + normalize_sessions(source_path).await?; + let target = GlobalDb::open_at(target_path) + .await + .ok_or_else(|| db_message("merge_sessions", "could not open target sessions DB"))?; + attach_as(target.conn(), source_path, "source").await?; + reject_session_content_collisions(target.conn()).await?; + target + .conn() + .execute("PRAGMA foreign_keys = OFF", ()) + .await + .map_err(|error| db_error("merge_sessions", error))?; + target + .conn() + .execute("BEGIN IMMEDIATE", ()) + .await + .map_err(|error| db_error("merge_sessions", error))?; + let result = merge_sessions_tx(target.conn(), offsets).await; + match result { + Ok(()) => target + .conn() + .execute("COMMIT", ()) + .await + .map_err(|error| db_error("merge_sessions", error))?, + Err(error) => { + let _ = target.conn().execute("ROLLBACK", ()).await; + let _ = target.conn().execute("DETACH DATABASE source", ()).await; + return Err(error); + } + }; + target + .conn() + .execute("DETACH DATABASE source", ()) + .await + .map_err(|error| db_error("merge_sessions", error))?; + crate::sessions::lcm::schema::rebuild_raw_fts(target.conn()) + .await + .ok_or_else(|| db_message("merge_sessions", "could not rebuild raw-message FTS"))?; + target.checkpoint().await; + target.close(); + Ok(()) +} + +async fn normalize_sessions(path: &Path) -> Result<()> { + let db = GlobalDb::open_at(path).await.ok_or_else(|| { + db_message( + "normalize_sessions", + format!("could not open '{}'", path.display()), + ) + })?; + crate::sessions::lcm::schema::ensure_lcm_schema(db.conn()) + .await + .map_err(|error| db_error("normalize_sessions", error))?; + crate::sessions::git_correlation::ensure_git_correlation_schema(db.conn()) + .await + .map_err(|error| db_error("normalize_sessions", error))?; + crate::sessions::workflow_index::ensure_workflow_index_schema(db.conn()) + .await + .map_err(|error| db_error("normalize_sessions", error))?; + db.conn() + .execute( + "CREATE TABLE IF NOT EXISTS session_backfill_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()) + )", + (), + ) + .await + .map_err(|error| db_error("normalize_sessions", error))?; + if !db.ensure_token_count_cache().await { + return Err(db_message( + "normalize_sessions", + "could not ensure dashboard token-count schema", + )); + } + db.checkpoint().await; + db.close(); + Ok(()) +} + +async fn reject_session_registry_rows(path: &Path) -> Result<()> { + let db = GlobalDb::open_read_only_at(path) + .await + .ok_or_else(|| db_message("merge_sessions", "could not inspect source sessions DB"))?; + for table in [ + "code_projects", + "project_aliases", + "store_instances", + "graph_scopes", + "store_artifacts", + ] { + if table_exists(db.conn(), "main", table).await? + && table_max_count(db.conn(), table).await? > 0 + { + return Err(db_message( + "merge_sessions", + format!("source sessions DB unexpectedly contains registry rows in {table}"), + )); + } + } + db.close(); + Ok(()) +} + +async fn reject_session_content_collisions(conn: &Connection) -> Result<()> { + for (label, sql) in [ + ( + "session message", + "SELECT COUNT(*) FROM source.session_messages s + JOIN session_messages t ON t.provider=s.provider AND t.message_id=s.message_id + WHERE t.session_id IS NOT s.session_id OR t.role IS NOT s.role + OR t.ordinal IS NOT s.ordinal OR t.text IS NOT s.text + OR t.kind IS NOT s.kind OR t.model IS NOT s.model", + ), + ( + "LCM raw message", + "SELECT COUNT(*) FROM source.lcm_raw_messages s + JOIN lcm_raw_messages t ON t.provider=s.provider AND t.message_id=s.message_id + WHERE t.session_id IS NOT s.session_id OR t.content_hash IS NOT s.content_hash + OR t.storage_kind IS NOT s.storage_kind OR t.payload_ref IS NOT s.payload_ref", + ), + ( + "LCM external payload", + "SELECT COUNT(*) FROM source.lcm_external_payloads s + JOIN lcm_external_payloads t ON t.payload_ref=s.payload_ref + WHERE t.content_hash IS NOT s.content_hash OR t.byte_count IS NOT s.byte_count", + ), + ( + "LCM summary node", + "SELECT COUNT(*) FROM source.lcm_summary_nodes s + JOIN lcm_summary_nodes t ON t.node_id=s.node_id + WHERE t.summary_hash IS NOT s.summary_hash OR t.summary_text IS NOT s.summary_text", + ), + ] { + let count = query_i64(conn, sql).await?; + if count > 0 { + return Err(db_message( + "merge_sessions", + format!( + "{count} divergent {label} collision(s); inputs and backups were preserved" + ), + )); + } + } + Ok(()) +} + +async fn merge_sessions_tx(conn: &Connection, offsets: &SessionMergeOffsets) -> Result<()> { + conn.execute_batch(&format!( + "CREATE TEMP TABLE IF NOT EXISTS consolidation_raw_map( + source_id INTEGER PRIMARY KEY, target_id INTEGER NOT NULL + ); + DELETE FROM consolidation_raw_map; + + INSERT OR IGNORE INTO projects(path, tokens_saved) + SELECT path, tokens_saved FROM source.projects; + UPDATE projects AS t SET tokens_saved = MAX(t.tokens_saved, COALESCE(( + SELECT s.tokens_saved FROM source.projects s WHERE s.path=t.path + ), t.tokens_saved)); + + INSERT OR IGNORE INTO turns( + message_id, project_hash, session_id, model, timestamp, input_tokens, + output_tokens, cache_write_tokens, cache_read_tokens, cost_usd, + category, tool_names + ) SELECT message_id, project_hash, session_id, model, timestamp, input_tokens, + output_tokens, cache_write_tokens, cache_read_tokens, cost_usd, + category, tool_names FROM source.turns; + + INSERT OR IGNORE INTO parse_offsets(file_path, byte_offset, mtime, file_id) + SELECT file_path, byte_offset, mtime, file_id FROM source.parse_offsets; + UPDATE parse_offsets AS t SET + byte_offset = CASE WHEN COALESCE((SELECT s.mtime FROM source.parse_offsets s WHERE s.file_path=t.file_path), -1) > t.mtime + THEN (SELECT s.byte_offset FROM source.parse_offsets s WHERE s.file_path=t.file_path) ELSE t.byte_offset END, + file_id = CASE WHEN COALESCE((SELECT s.mtime FROM source.parse_offsets s WHERE s.file_path=t.file_path), -1) > t.mtime + THEN (SELECT s.file_id FROM source.parse_offsets s WHERE s.file_path=t.file_path) ELSE t.file_id END, + mtime = MAX(t.mtime, COALESCE((SELECT s.mtime FROM source.parse_offsets s WHERE s.file_path=t.file_path), t.mtime)); + + INSERT OR IGNORE INTO savings_ledger(id, ts, project_path, tool_name, before_tokens, after_tokens) + SELECT id + {savings}, ts, project_path, tool_name, before_tokens, after_tokens + FROM source.savings_ledger; + INSERT OR IGNORE INTO analytics_events( + id, provider, project_id, session_id, timestamp, event_kind, hook_name, + tool_name, tool_category, skill_name, hint_category, hint_id, outcome, metadata_json + ) SELECT id + {analytics}, provider, project_id, session_id, timestamp, + event_kind, hook_name, tool_name, tool_category, skill_name, hint_category, + hint_id, outcome, metadata_json FROM source.analytics_events; + + INSERT OR IGNORE INTO sessions( + provider, session_id, project_key, project_path, title, started_at, + ended_at, transcript_path, metadata_json, parent_session_id, + is_subagent, agent_id, parent_tool_use_id + ) SELECT provider, session_id, project_key, project_path, title, started_at, + ended_at, transcript_path, metadata_json, parent_session_id, + is_subagent, agent_id, parent_tool_use_id FROM source.sessions; + UPDATE sessions AS t SET + started_at = CASE + WHEN t.started_at IS NULL THEN (SELECT s.started_at FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id) + WHEN (SELECT s.started_at FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id) IS NULL THEN t.started_at + ELSE MIN(t.started_at, (SELECT s.started_at FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id)) END, + ended_at = CASE + WHEN t.ended_at IS NULL THEN (SELECT s.ended_at FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id) + WHEN (SELECT s.ended_at FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id) IS NULL THEN t.ended_at + ELSE MAX(t.ended_at, (SELECT s.ended_at FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id)) END, + title = COALESCE(t.title, (SELECT s.title FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id)), + transcript_path = COALESCE(t.transcript_path, (SELECT s.transcript_path FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id)), + metadata_json = COALESCE(t.metadata_json, (SELECT s.metadata_json FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id)), + parent_session_id = COALESCE(t.parent_session_id, (SELECT s.parent_session_id FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id)), + is_subagent = MAX(t.is_subagent, COALESCE((SELECT s.is_subagent FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id), 0)), + agent_id = COALESCE(t.agent_id, (SELECT s.agent_id FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id)), + parent_tool_use_id = COALESCE(t.parent_tool_use_id, (SELECT s.parent_tool_use_id FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id)) + WHERE EXISTS (SELECT 1 FROM source.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id); + + INSERT OR IGNORE INTO session_messages( + provider, message_id, session_id, role, timestamp, ordinal, text, kind, + model, tool_names, source_path, source_offset, metadata_json + ) SELECT provider, message_id, session_id, role, timestamp, ordinal, text, kind, + model, tool_names, source_path, source_offset, metadata_json + FROM source.session_messages; + + INSERT OR IGNORE INTO session_schema_migrations(name, version, applied_at) + SELECT name, version, applied_at FROM source.session_schema_migrations; + UPDATE session_schema_migrations AS t SET + version = MAX(t.version, COALESCE((SELECT s.version FROM source.session_schema_migrations s WHERE s.name=t.name), t.version)), + applied_at = MAX(t.applied_at, COALESCE((SELECT s.applied_at FROM source.session_schema_migrations s WHERE s.name=t.name), t.applied_at)); + + INSERT OR IGNORE INTO lcm_raw_messages( + provider, message_id, session_id, store_id, role, ordinal, timestamp, + content, content_hash, storage_kind, payload_ref, snippet_text, index_text, + legacy_source, legacy_truncated, metadata_json + ) SELECT provider, message_id, session_id, store_id + {raw}, role, ordinal, + timestamp, content, content_hash, storage_kind, payload_ref, snippet_text, + index_text, legacy_source, legacy_truncated, metadata_json + FROM source.lcm_raw_messages; + INSERT INTO consolidation_raw_map(source_id, target_id) + SELECT s.store_id, t.store_id FROM source.lcm_raw_messages s + JOIN lcm_raw_messages t ON t.provider=s.provider AND t.message_id=s.message_id; + + INSERT OR IGNORE INTO lcm_external_payloads( + payload_ref, provider, session_id, message_id, kind, content_hash, + byte_count, char_count, created_at, metadata_json + ) SELECT payload_ref, provider, session_id, message_id, kind, content_hash, + byte_count, char_count, created_at, metadata_json FROM source.lcm_external_payloads; + INSERT OR IGNORE INTO lcm_gc_marks(payload_ref, state, first_seen_at, updated_at) + SELECT payload_ref, state, first_seen_at, updated_at FROM source.lcm_gc_marks; + INSERT OR IGNORE INTO lcm_gc_meta(key, value) SELECT key, value FROM source.lcm_gc_meta; + INSERT OR IGNORE INTO lcm_summary_nodes( + node_id, provider, conversation_id, session_id, depth, summary_text, + summary_hash, summary_token_count, source_token_count, source_time_start, + source_time_end, expand_hint, metadata_json, created_at + ) SELECT node_id, provider, conversation_id, session_id, depth, summary_text, + summary_hash, summary_token_count, source_token_count, source_time_start, + source_time_end, expand_hint, metadata_json, created_at FROM source.lcm_summary_nodes; + INSERT OR IGNORE INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) + SELECT s.node_id, s.source_kind, + CASE WHEN s.source_kind='raw_message' THEN CAST(( + SELECT target_id FROM consolidation_raw_map + WHERE source_id=CAST(s.source_id AS INTEGER) + ) AS TEXT) ELSE s.source_id END, + s.ordinal + FROM source.lcm_summary_sources s; + + INSERT OR REPLACE INTO lcm_lifecycle_state( + provider, conversation_id, current_session_id, last_finalized_session_id, + current_frontier_store_id, last_finalized_frontier_store_id, rollover_at, + reset_at, maintenance_at, boundary_skip_at, updated_at + ) SELECT s.provider, s.conversation_id, s.current_session_id, + s.last_finalized_session_id, + (SELECT target_id FROM consolidation_raw_map WHERE source_id=s.current_frontier_store_id), + (SELECT target_id FROM consolidation_raw_map WHERE source_id=s.last_finalized_frontier_store_id), + s.rollover_at, s.reset_at, s.maintenance_at, s.boundary_skip_at, s.updated_at + FROM source.lcm_lifecycle_state s + WHERE NOT EXISTS ( + SELECT 1 FROM lcm_lifecycle_state t + WHERE t.provider=s.provider AND t.conversation_id=s.conversation_id + AND t.updated_at >= s.updated_at + ); + INSERT OR IGNORE INTO lcm_maintenance_debt( + provider, conversation_id, debt_id, debt_kind, from_store_id, + to_store_id, metadata_json, created_at + ) SELECT s.provider, s.conversation_id, s.debt_id, s.debt_kind, + (SELECT target_id FROM consolidation_raw_map WHERE source_id=s.from_store_id), + (SELECT target_id FROM consolidation_raw_map WHERE source_id=s.to_store_id), + s.metadata_json, s.created_at FROM source.lcm_maintenance_debt s; + + INSERT OR IGNORE INTO workflow_runs( + run_id, parent_session_id, name, description, phase_json, status, + started_ts, ended_ts, result_summary, agent_count, created_at, updated_at + ) SELECT run_id, parent_session_id, name, description, phase_json, status, + started_ts, ended_ts, result_summary, agent_count, created_at, updated_at + FROM source.workflow_runs; + INSERT OR REPLACE INTO workflow_runs( + run_id, parent_session_id, name, description, phase_json, status, + started_ts, ended_ts, result_summary, agent_count, created_at, updated_at + ) SELECT s.run_id, s.parent_session_id, s.name, s.description, s.phase_json, + s.status, s.started_ts, s.ended_ts, s.result_summary, s.agent_count, + s.created_at, s.updated_at FROM source.workflow_runs s + WHERE EXISTS (SELECT 1 FROM workflow_runs t WHERE t.run_id=s.run_id AND t.updated_at < s.updated_at); + INSERT OR IGNORE INTO workflow_agents( + run_id, agent_label, agent_id, phase, transcript_path, agent_session_id, + status, model, tokens, started_ts, ended_ts, created_at, updated_at + ) SELECT run_id, agent_label, agent_id, phase, transcript_path, agent_session_id, + status, model, tokens, started_ts, ended_ts, created_at, updated_at + FROM source.workflow_agents; + INSERT OR REPLACE INTO workflow_agents( + run_id, agent_label, agent_id, phase, transcript_path, agent_session_id, + status, model, tokens, started_ts, ended_ts, created_at, updated_at + ) SELECT s.run_id, s.agent_label, s.agent_id, s.phase, s.transcript_path, + s.agent_session_id, s.status, s.model, s.tokens, s.started_ts, s.ended_ts, + s.created_at, s.updated_at FROM source.workflow_agents s + WHERE EXISTS ( + SELECT 1 FROM workflow_agents t + WHERE t.run_id=s.run_id AND t.agent_label=s.agent_label AND t.agent_id=s.agent_id + AND t.updated_at < s.updated_at + ); + INSERT OR IGNORE INTO workflow_index_meta(key, value, updated_at) + SELECT key, value, updated_at FROM source.workflow_index_meta; + UPDATE workflow_index_meta AS t SET + value = MAX(t.value, COALESCE((SELECT s.value FROM source.workflow_index_meta s WHERE s.key=t.key), t.value)), + updated_at = MAX(t.updated_at, COALESCE((SELECT s.updated_at FROM source.workflow_index_meta s WHERE s.key=t.key), t.updated_at)); + + INSERT OR IGNORE INTO session_git_spans( + span_id, provider, session_id, thread_id, branch, worktree, first_ts, + last_ts, event_count, source, created_at, updated_at + ) SELECT span_id + {span}, provider, session_id, thread_id, branch, worktree, + first_ts, last_ts, event_count, source, created_at, updated_at + FROM source.session_git_spans; + INSERT OR IGNORE INTO commit_sessions( + commit_sha, provider, session_id, branch, worktree, committed_at, + span_overlap_kind, span_id, relation, evidence, confidence, + evidence_message_id, created_at + ) SELECT commit_sha, provider, session_id, branch, worktree, committed_at, + span_overlap_kind, CASE WHEN span_id IS NULL THEN NULL ELSE span_id + {span} END, + relation, evidence, confidence, evidence_message_id, created_at + FROM source.commit_sessions; + UPDATE commit_sessions AS t SET + branch = (SELECT s.branch FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id), + worktree = (SELECT s.worktree FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id), + committed_at = (SELECT s.committed_at FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id), + span_overlap_kind = (SELECT s.span_overlap_kind FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id), + span_id = (SELECT CASE WHEN s.span_id IS NULL THEN NULL ELSE s.span_id + {span} END FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id), + relation = (SELECT s.relation FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id), + evidence = (SELECT s.evidence FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id), + confidence = (SELECT s.confidence FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id), + evidence_message_id = (SELECT s.evidence_message_id FROM source.commit_sessions s WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id) + WHERE EXISTS ( + SELECT 1 FROM source.commit_sessions s + WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id + AND s.confidence > t.confidence + ); + INSERT OR IGNORE INTO git_correlation_meta(key, value, updated_at) + SELECT key, value, updated_at FROM source.git_correlation_meta; + UPDATE git_correlation_meta AS t SET + value = MAX(t.value, COALESCE((SELECT s.value FROM source.git_correlation_meta s WHERE s.key=t.key), t.value)), + updated_at = MAX(t.updated_at, COALESCE((SELECT s.updated_at FROM source.git_correlation_meta s WHERE s.key=t.key), t.updated_at)); + + INSERT OR IGNORE INTO session_backfill_meta(key, value, updated_at) + SELECT key, value, updated_at FROM source.session_backfill_meta; + UPDATE session_backfill_meta AS t SET + value = MAX(t.value, COALESCE((SELECT s.value FROM source.session_backfill_meta s WHERE s.key=t.key), t.value)), + updated_at = MAX(t.updated_at, COALESCE((SELECT s.updated_at FROM source.session_backfill_meta s WHERE s.key=t.key), t.updated_at)); + + INSERT OR IGNORE INTO dashboard_token_counts( + store, provider, message_id, text_len, encoder, token_count, computed_at + ) SELECT store, provider, message_id, text_len, encoder, token_count, computed_at + FROM source.dashboard_token_counts;", + raw = offsets.raw, + span = offsets.span, + savings = offsets.savings, + analytics = offsets.analytics, + )) + .await + .map_err(|error| db_error("merge_sessions", error))?; + Ok(()) +} + +async fn attach(conn: &Connection, path: &Path) -> Result<()> { + attach_as(conn, path, "other").await +} + +async fn attach_as(conn: &Connection, path: &Path, alias: &str) -> Result<()> { + let sql = format!("ATTACH DATABASE ?1 AS {}", quote_identifier(alias)); + conn.execute(&sql, params![path.to_string_lossy().to_string()]) + .await + .map_err(|error| db_error("attach_database", error))?; + Ok(()) +} + +async fn table_exists(conn: &Connection, schema: &str, table: &str) -> Result { + let sql = format!( + "SELECT COUNT(*) FROM {}.sqlite_schema WHERE type='table' AND name=?1", + quote_identifier(schema) + ); + let mut rows = conn + .query(&sql, params![table]) + .await + .map_err(|error| db_error("table_exists", error))?; + let row = rows + .next() + .await + .map_err(|error| db_error("table_exists", error))? + .ok_or_else(|| db_message("table_exists", "table probe returned no row"))?; + Ok(row + .get::(0) + .map_err(|error| db_error("table_exists", error))? + > 0) +} + +async fn table_max(conn: &Connection, table: &str, column: &str) -> Result { + if !table_exists(conn, "main", table).await? { + return Ok(0); + } + query_i64( + conn, + &format!( + "SELECT COALESCE(MAX({}), 0) FROM {}", + quote_identifier(column), + quote_identifier(table) + ), + ) + .await +} + +async fn db_table_max(path: &Path, table: &str, column: &str) -> Result { + let db = GlobalDb::open_read_only_at(path) + .await + .ok_or_else(|| db_message("table_max", format!("could not open '{}'", path.display())))?; + let value = table_max(db.conn(), table, column).await?; + db.close(); + Ok(value) +} + +async fn table_max_count(conn: &Connection, table: &str) -> Result { + query_i64( + conn, + &format!("SELECT COUNT(*) FROM {}", quote_identifier(table)), + ) + .await +} + +async fn query_i64(conn: &Connection, sql: &str) -> Result { + let mut rows = conn + .query(sql, ()) + .await + .map_err(|error| db_error("query_i64", error))?; + let row = rows + .next() + .await + .map_err(|error| db_error("query_i64", error))? + .ok_or_else(|| db_message("query_i64", "query returned no row"))?; + row.get::(0) + .map_err(|error| db_error("query_i64", error)) +} + +fn checked_advance(base: i64, source_max: i64, label: &str) -> Result { + base.checked_add(source_max) + .and_then(|value| value.checked_add(1)) + .ok_or_else(|| db_message("plan_offsets", format!("{label} offset overflow"))) +} + +fn quote_identifier(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\"\"")) +} + +fn db_error(operation: &str, error: impl std::fmt::Display) -> TraceDecayError { + TraceDecayError::Database { + message: error.to_string(), + operation: operation.to_string(), + } +} + +fn db_message(operation: &str, message: impl Into) -> TraceDecayError { + TraceDecayError::Database { + message: message.into(), + operation: operation.to_string(), + } +} diff --git a/src/migrate/consolidate/sqlite/inspect.rs b/src/migrate/consolidate/sqlite/inspect.rs new file mode 100644 index 00000000..4f1243bb --- /dev/null +++ b/src/migrate/consolidate/sqlite/inspect.rs @@ -0,0 +1,337 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use libsql::{Builder, Connection, Database as LibsqlDatabase}; + +use super::{attach, db_error, db_message, query_i64, quote_identifier, table_exists}; +use crate::errors::Result; + +#[derive(Debug, Clone, Copy)] +pub(in crate::migrate::consolidate) struct DatabaseCollisionCounts { + pub sessions: u64, + pub messages: u64, + pub lcm_messages: u64, +} + +pub(in crate::migrate::consolidate) struct OfflineDatabaseGuards { + _holds: Vec<(Connection, LibsqlDatabase)>, +} + +#[derive(Default)] +pub(in crate::migrate::consolidate) struct GraphLogicalIdentities { + facts: HashSet>, + feedback: HashSet>, +} + +impl GraphLogicalIdentities { + pub(in crate::migrate::consolidate) fn fact_count(&self) -> u64 { + self.facts.len() as u64 + } + + pub(in crate::migrate::consolidate) fn feedback_count(&self) -> u64 { + self.feedback.len() as u64 + } + + pub(in crate::migrate::consolidate) fn fact_overlap(&self, other: &Self) -> u64 { + self.facts.intersection(&other.facts).count() as u64 + } + + pub(in crate::migrate::consolidate) fn facts_union_matches( + &self, + other: &Self, + destination: &Self, + ) -> bool { + destination.facts.len() == self.facts.union(&other.facts).count() + && destination + .facts + .iter() + .all(|key| self.facts.contains(key) || other.facts.contains(key)) + } + + pub(in crate::migrate::consolidate) fn feedback_union_matches( + &self, + other: &Self, + destination: &Self, + ) -> bool { + destination.feedback.len() == self.feedback.union(&other.feedback).count() + && destination + .feedback + .iter() + .all(|key| self.feedback.contains(key) || other.feedback.contains(key)) + } +} + +pub(in crate::migrate::consolidate) async fn extend_graph_identities( + conn: &Connection, + identities: &mut GraphLogicalIdentities, +) -> Result<()> { + if table_exists(conn, "main", "memory_facts").await? { + read_fact_keys(conn, &mut identities.facts).await?; + } + if table_exists(conn, "main", "memory_feedback_events").await? { + read_feedback_keys(conn, &mut identities.feedback).await?; + } + Ok(()) +} + +pub(in crate::migrate::consolidate) async fn acquire_offline_guards( + paths: &[PathBuf], +) -> Result { + let mut ordered = paths.to_vec(); + ordered.sort(); + ordered.dedup(); + let mut holds = Vec::new(); + for path in ordered { + if !crate::storage::has_sqlite_database_header(&path).map_err(|error| { + db_error( + "acquire_offline_guards", + format!("failed to inspect '{}': {error}", path.display()), + ) + })? { + return Err(db_message( + "acquire_offline_guards", + format!("file is not a database: '{}'", path.display()), + )); + } + let db = Builder::new_local(&path) + .build() + .await + .map_err(|error| db_error("acquire_offline_guards", error))?; + let conn = db + .connect() + .map_err(|error| db_error("acquire_offline_guards", error))?; + conn.execute_batch("PRAGMA busy_timeout = 0;") + .await + .map_err(|error| db_error("acquire_offline_guards", error))?; + conn.execute("BEGIN IMMEDIATE", ()).await.map_err(|error| { + db_message( + "acquire_offline_guards", + format!( + "database '{}' is still writable by another process: {error}; stop MCP/CLI writers and retry", + path.display() + ), + ) + })?; + holds.push((conn, db)); + } + Ok(OfflineDatabaseGuards { _holds: holds }) +} + +async fn read_fact_keys(conn: &Connection, identities: &mut HashSet>) -> Result<()> { + let mut rows = conn + .query("SELECT content FROM memory_facts", ()) + .await + .map_err(|error| db_error("logical_identities", error))?; + while let Some(row) = rows + .next() + .await + .map_err(|error| db_error("logical_identities", error))? + { + let mut key = Vec::new(); + push_text( + &mut key, + &row.get::(0) + .map_err(|error| db_error("logical_identities", error))?, + ); + identities.insert(key); + } + Ok(()) +} + +async fn read_feedback_keys(conn: &Connection, identities: &mut HashSet>) -> Result<()> { + let mut rows = conn + .query( + "SELECT f.content, e.action, e.trust_delta, e.old_trust, + e.new_trust, e.created_at, e.source, e.note + FROM memory_feedback_events e + JOIN memory_facts f ON f.fact_id=e.fact_id", + (), + ) + .await + .map_err(|error| db_error("logical_identities", error))?; + while let Some(row) = rows + .next() + .await + .map_err(|error| db_error("logical_identities", error))? + { + let mut key = Vec::new(); + push_text(&mut key, &row_text(&row, 0)?); + push_text(&mut key, &row_text(&row, 1)?); + for index in 2..5 { + push_f64(&mut key, row.get::(index).map_err(logical_error)?); + } + key.extend_from_slice(&row.get::(5).map_err(logical_error)?.to_be_bytes()); + push_text(&mut key, &row_text(&row, 6)?); + match row.get::>(7).map_err(logical_error)? { + Some(note) => { + key.push(1); + push_text(&mut key, ¬e); + } + None => key.push(0), + } + identities.insert(key); + } + Ok(()) +} + +fn row_text(row: &libsql::Row, index: i32) -> Result { + row.get::(index).map_err(logical_error) +} + +fn logical_error(error: libsql::Error) -> crate::errors::TraceDecayError { + db_error("logical_identities", error) +} + +fn push_text(target: &mut Vec, value: &str) { + target.extend_from_slice(&(value.len() as u64).to_be_bytes()); + target.extend_from_slice(value.as_bytes()); +} + +fn push_f64(target: &mut Vec, value: f64) { + let normalized = if value == 0.0 { 0.0 } else { value }; + target.extend_from_slice(&normalized.to_bits().to_be_bytes()); +} + +#[cfg(test)] +pub(in crate::migrate::consolidate) async fn count_rows(path: &Path, table: &str) -> Result { + let snapshots = crate::sqlite_read_snapshot::SnapshotSet::capture(&[path.to_path_buf()]) + .await + .map_err(|error| db_error("read_snapshot", error))?; + count_rows_in(&snapshots, path, table).await +} + +pub(in crate::migrate::consolidate) async fn count_rows_in( + snapshots: &crate::sqlite_read_snapshot::SnapshotSet, + path: &Path, + table: &str, +) -> Result { + if !path.is_file() { + return Ok(0); + } + let db = read_snapshot(snapshots, path)?; + let count = if table_exists(db.connection(), "main", table).await? { + query_i64( + db.connection(), + &format!("SELECT COUNT(*) FROM {}", quote_identifier(table)), + ) + .await? + } else { + 0 + }; + u64::try_from(count).map_err(|error| db_error("count_rows", error)) +} + +pub(in crate::migrate::consolidate) async fn quick_check_in( + snapshots: &crate::sqlite_read_snapshot::SnapshotSet, + path: &Path, +) -> Result<()> { + let db = read_snapshot(snapshots, path)?; + quick_check_connection(db.connection(), path).await +} + +pub(in crate::migrate::consolidate) async fn quick_check_connection( + conn: &Connection, + path: &Path, +) -> Result<()> { + let mut rows = conn + .query("PRAGMA quick_check", ()) + .await + .map_err(|error| db_error("quick_check", error))?; + let row = rows + .next() + .await + .map_err(|error| db_error("quick_check", error))? + .ok_or_else(|| db_message("quick_check", "quick_check returned no row"))?; + let value = row + .get::(0) + .map_err(|error| db_error("quick_check", error))?; + if value != "ok" { + return Err(db_message( + "quick_check", + format!( + "SQLite quick_check failed for '{}': {value}", + path.display() + ), + )); + } + Ok(()) +} + +pub(in crate::migrate::consolidate) async fn inspect_collisions( + snapshots: &crate::sqlite_read_snapshot::SnapshotSet, + source_sessions: &Path, + target_sessions: &Path, +) -> Result { + let sessions = overlap_count( + snapshots, + source_sessions, + target_sessions, + "sessions", + "s.provider = t.provider AND s.session_id = t.session_id", + ) + .await?; + let messages = overlap_count( + snapshots, + source_sessions, + target_sessions, + "session_messages", + "s.provider = t.provider AND s.message_id = t.message_id", + ) + .await?; + let lcm_messages = overlap_count( + snapshots, + source_sessions, + target_sessions, + "lcm_raw_messages", + "s.provider = t.provider AND s.message_id = t.message_id", + ) + .await?; + Ok(DatabaseCollisionCounts { + sessions, + messages, + lcm_messages, + }) +} + +async fn overlap_count( + snapshots: &crate::sqlite_read_snapshot::SnapshotSet, + source: &Path, + target: &Path, + table: &str, + join: &str, +) -> Result { + if !source.is_file() || !target.is_file() { + return Ok(0); + } + let source = read_snapshot(snapshots, source)?; + let target = read_snapshot(snapshots, target)?; + let conn = source.connection(); + attach(conn, target.path()).await?; + let count = + if table_exists(conn, "main", table).await? && table_exists(conn, "other", table).await? { + query_i64( + conn, + &format!( + "SELECT COUNT(*) FROM main.{} s JOIN other.{} t ON {join}", + quote_identifier(table), + quote_identifier(table) + ), + ) + .await? + } else { + 0 + }; + conn.execute("DETACH DATABASE other", ()) + .await + .map_err(|error| db_error("inspect_collisions", error))?; + u64::try_from(count).map_err(|error| db_error("inspect_collisions", error)) +} + +fn read_snapshot<'a>( + snapshots: &'a crate::sqlite_read_snapshot::SnapshotSet, + path: &Path, +) -> Result<&'a crate::sqlite_read_snapshot::SnapshotDatabase> { + snapshots + .get(path) + .map_err(|error| db_error("read_snapshot", error)) +} diff --git a/src/migrate/consolidate/sqlite/verify.rs b/src/migrate/consolidate/sqlite/verify.rs new file mode 100644 index 00000000..33dbb941 --- /dev/null +++ b/src/migrate/consolidate/sqlite/verify.rs @@ -0,0 +1,596 @@ +use std::path::Path; + +use libsql::Connection; + +use super::{SessionMergeOffsets, attach_as, db_error, db_message, query_i64}; +use crate::errors::Result; + +struct TableVerification { + label: &'static str, + table: &'static str, + columns: &'static str, + expected: String, +} + +pub(in crate::migrate::consolidate) async fn verify_session_union_sql( + input_snapshots: &crate::sqlite_read_snapshot::SnapshotSet, + source: &Path, + target: &Path, + destination_snapshots: &crate::sqlite_read_snapshot::SnapshotSet, + destination: &Path, + destination_root: &Path, + offsets: &SessionMergeOffsets, +) -> Result<()> { + let conn = destination_snapshots.get(destination).map_err(|error| { + db_error( + "verify_consolidation", + format!("could not read destination snapshot: {error}"), + ) + })?; + let conn = conn.connection(); + conn.execute_batch("PRAGMA temp_store=FILE; PRAGMA cache_size=-32768;") + .await + .map_err(|error| db_error("verify_consolidation", error))?; + attach_as( + conn, + input_snapshots + .get(source) + .map_err(|error| db_error("verify_consolidation", error))? + .path(), + "source_input", + ) + .await?; + attach_as( + conn, + input_snapshots + .get(target) + .map_err(|error| db_error("verify_consolidation", error))? + .path(), + "target_input", + ) + .await?; + + let result = match verify_attached_tables(conn, offsets).await { + Ok(()) => verify_payload_files(conn, destination_root).await, + Err(error) => Err(error), + }; + let _ = conn.execute("DETACH DATABASE source_input", ()).await; + let _ = conn.execute("DETACH DATABASE target_input", ()).await; + result +} + +async fn verify_attached_tables(conn: &Connection, offsets: &SessionMergeOffsets) -> Result<()> { + for spec in verification_specs(offsets) { + verify_table(conn, &spec).await?; + } + for (label, backing, fts) in [ + ( + "session message FTS", + "session_messages", + "session_messages_fts", + ), + ( + "LCM raw-message FTS", + "lcm_raw_messages", + "lcm_raw_messages_fts", + ), + ( + "LCM summary-node FTS", + "lcm_summary_nodes", + "lcm_summary_nodes_fts", + ), + ] { + let sql = format!( + "SELECT + (SELECT COUNT(*) FROM (SELECT rowid FROM {backing} EXCEPT SELECT rowid FROM {fts})) + + (SELECT COUNT(*) FROM (SELECT rowid FROM {fts} EXCEPT SELECT rowid FROM {backing}))" + ); + let differences = query_i64(conn, &sql).await?; + if differences != 0 { + return Err(db_message( + "verify_consolidation", + format!("destination {label} differs from its durable backing table"), + )); + } + } + Ok(()) +} + +async fn verify_table(conn: &Connection, spec: &TableVerification) -> Result<()> { + let sql = format!( + "WITH expected AS ({expected}) + SELECT + (SELECT COUNT(*) FROM ( + SELECT * FROM expected + EXCEPT SELECT {columns} FROM main.{table} + )) + + (SELECT COUNT(*) FROM ( + SELECT {columns} FROM main.{table} + EXCEPT SELECT * FROM expected + ))", + expected = spec.expected, + columns = spec.columns, + table = spec.table, + ); + let differences = query_i64(conn, &sql).await?; + if differences != 0 { + return Err(db_message( + "verify_consolidation", + format!( + "destination {} logical union differs from frozen inputs: {differences} difference(s)", + spec.label + ), + )); + } + Ok(()) +} + +fn verification_specs(offsets: &SessionMergeOffsets) -> Vec { + let mut specs = vec![ + custom( + "project accounting", + "projects", + "path, tokens_saved", + "SELECT path, MAX(tokens_saved) AS tokens_saved FROM ( + SELECT path, tokens_saved FROM target_input.projects + UNION ALL SELECT path, tokens_saved FROM source_input.projects + ) GROUP BY path", + ), + target_wins( + "turn", + "turns", + "message_id, project_hash, session_id, model, timestamp, input_tokens, output_tokens, cache_write_tokens, cache_read_tokens, cost_usd, category, tool_names", + "t.message_id=s.message_id", + ), + custom( + "parse offset", + "parse_offsets", + "file_path, byte_offset, mtime, file_id", + "SELECT t.file_path, + CASE WHEN s.mtime > t.mtime THEN s.byte_offset ELSE t.byte_offset END, + MAX(t.mtime, s.mtime), + CASE WHEN s.mtime > t.mtime THEN s.file_id ELSE t.file_id END + FROM target_input.parse_offsets t + JOIN source_input.parse_offsets s ON s.file_path=t.file_path + UNION ALL + SELECT t.file_path, t.byte_offset, t.mtime, t.file_id + FROM target_input.parse_offsets t + WHERE NOT EXISTS (SELECT 1 FROM source_input.parse_offsets s WHERE s.file_path=t.file_path) + UNION ALL + SELECT s.file_path, s.byte_offset, s.mtime, s.file_id + FROM source_input.parse_offsets s + WHERE NOT EXISTS (SELECT 1 FROM target_input.parse_offsets t WHERE t.file_path=s.file_path)", + ), + offset_append( + "savings ledger", + "savings_ledger", + "id, ts, project_path, tool_name, before_tokens, after_tokens", + &format!( + "id + {}, ts, project_path, tool_name, before_tokens, after_tokens", + offsets.savings + ), + ), + offset_append( + "analytics event", + "analytics_events", + "id, provider, project_id, session_id, timestamp, event_kind, hook_name, tool_name, tool_category, skill_name, hint_category, hint_id, outcome, metadata_json", + &format!( + "id + {}, provider, project_id, session_id, timestamp, event_kind, hook_name, tool_name, tool_category, skill_name, hint_category, hint_id, outcome, metadata_json", + offsets.analytics + ), + ), + custom( + "session", + "sessions", + "provider, session_id, project_key, project_path, title, started_at, ended_at, transcript_path, metadata_json, parent_session_id, is_subagent, agent_id, parent_tool_use_id", + "SELECT t.provider, t.session_id, t.project_key, t.project_path, + COALESCE(t.title, s.title), + CASE WHEN t.started_at IS NULL THEN s.started_at + WHEN s.started_at IS NULL THEN t.started_at + ELSE MIN(t.started_at, s.started_at) END, + CASE WHEN t.ended_at IS NULL THEN s.ended_at + WHEN s.ended_at IS NULL THEN t.ended_at + ELSE MAX(t.ended_at, s.ended_at) END, + COALESCE(t.transcript_path, s.transcript_path), + COALESCE(t.metadata_json, s.metadata_json), + COALESCE(t.parent_session_id, s.parent_session_id), + MAX(t.is_subagent, s.is_subagent), + COALESCE(t.agent_id, s.agent_id), + COALESCE(t.parent_tool_use_id, s.parent_tool_use_id) + FROM target_input.sessions t + JOIN source_input.sessions s ON s.provider=t.provider AND s.session_id=t.session_id + UNION ALL + SELECT t.provider, t.session_id, t.project_key, t.project_path, t.title, + t.started_at, t.ended_at, t.transcript_path, t.metadata_json, + t.parent_session_id, t.is_subagent, t.agent_id, t.parent_tool_use_id + FROM target_input.sessions t + WHERE NOT EXISTS (SELECT 1 FROM source_input.sessions s WHERE s.provider=t.provider AND s.session_id=t.session_id) + UNION ALL + SELECT s.provider, s.session_id, s.project_key, s.project_path, s.title, + s.started_at, s.ended_at, s.transcript_path, s.metadata_json, + s.parent_session_id, s.is_subagent, s.agent_id, s.parent_tool_use_id + FROM source_input.sessions s + WHERE NOT EXISTS (SELECT 1 FROM target_input.sessions t WHERE t.provider=s.provider AND t.session_id=s.session_id)", + ), + target_wins( + "session message", + "session_messages", + "provider, message_id, session_id, role, timestamp, ordinal, text, kind, model, tool_names, source_path, source_offset, metadata_json", + "t.provider=s.provider AND t.message_id=s.message_id", + ), + custom( + "session schema migration", + "session_schema_migrations", + "name, version, applied_at", + "SELECT name, MAX(version), MAX(applied_at) FROM ( + SELECT name, version, applied_at FROM target_input.session_schema_migrations + UNION ALL SELECT name, version, applied_at FROM source_input.session_schema_migrations + ) GROUP BY name", + ), + projected_target_wins( + "LCM raw message", + "lcm_raw_messages", + "provider, message_id, session_id, store_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, snippet_text, index_text, legacy_source, legacy_truncated, metadata_json", + &format!( + "provider, message_id, session_id, store_id + {}, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, snippet_text, index_text, legacy_source, legacy_truncated, metadata_json", + offsets.raw + ), + "t.provider=s.provider AND t.message_id=s.message_id", + ), + target_wins( + "LCM external payload", + "lcm_external_payloads", + "payload_ref, provider, session_id, message_id, kind, content_hash, byte_count, char_count, created_at, metadata_json", + "t.payload_ref=s.payload_ref", + ), + target_wins( + "LCM GC mark", + "lcm_gc_marks", + "payload_ref, state, first_seen_at, updated_at", + "t.payload_ref=s.payload_ref", + ), + target_wins("LCM GC metadata", "lcm_gc_meta", "key, value", "t.key=s.key"), + target_wins( + "LCM summary node", + "lcm_summary_nodes", + "node_id, provider, conversation_id, session_id, depth, summary_text, summary_hash, summary_token_count, source_token_count, source_time_start, source_time_end, expand_hint, metadata_json, created_at", + "t.node_id=s.node_id", + ), + projected_target_wins( + "LCM summary source", + "lcm_summary_sources", + "node_id, source_kind, source_id, ordinal", + &format!( + "s.node_id, s.source_kind, + CASE WHEN s.source_kind='raw_message' THEN CAST({} AS TEXT) + ELSE s.source_id END, + s.ordinal", + remapped_raw_id("CAST(s.source_id AS INTEGER)", offsets.raw) + ), + "t.node_id=s.node_id AND t.ordinal=s.ordinal", + ), + ]; + + let current_frontier = remapped_raw_id("s.current_frontier_store_id", offsets.raw); + let finalized_frontier = remapped_raw_id("s.last_finalized_frontier_store_id", offsets.raw); + specs.push(custom_owned( + "LCM lifecycle state", + "lcm_lifecycle_state", + "provider, conversation_id, current_session_id, last_finalized_session_id, current_frontier_store_id, last_finalized_frontier_store_id, rollover_at, reset_at, maintenance_at, boundary_skip_at, updated_at", + format!( + "SELECT t.provider, t.conversation_id, t.current_session_id, + t.last_finalized_session_id, t.current_frontier_store_id, + t.last_finalized_frontier_store_id, t.rollover_at, t.reset_at, + t.maintenance_at, t.boundary_skip_at, t.updated_at + FROM target_input.lcm_lifecycle_state t + WHERE NOT EXISTS ( + SELECT 1 FROM source_input.lcm_lifecycle_state s + WHERE s.provider=t.provider AND s.conversation_id=t.conversation_id + AND s.updated_at > t.updated_at + ) + UNION ALL + SELECT s.provider, s.conversation_id, s.current_session_id, + s.last_finalized_session_id, {current_frontier}, {finalized_frontier}, + s.rollover_at, s.reset_at, s.maintenance_at, s.boundary_skip_at, + s.updated_at + FROM source_input.lcm_lifecycle_state s + WHERE NOT EXISTS ( + SELECT 1 FROM target_input.lcm_lifecycle_state t + WHERE t.provider=s.provider AND t.conversation_id=s.conversation_id + AND t.updated_at >= s.updated_at + )" + ), + )); + + let from_store = remapped_raw_id("s.from_store_id", offsets.raw); + let to_store = remapped_raw_id("s.to_store_id", offsets.raw); + specs.push(custom_owned( + "LCM maintenance debt", + "lcm_maintenance_debt", + "provider, conversation_id, debt_id, debt_kind, from_store_id, to_store_id, metadata_json, created_at", + format!( + "SELECT provider, conversation_id, debt_id, debt_kind, from_store_id, + to_store_id, metadata_json, created_at + FROM target_input.lcm_maintenance_debt + UNION ALL + SELECT s.provider, s.conversation_id, s.debt_id, s.debt_kind, + {from_store}, {to_store}, s.metadata_json, s.created_at + FROM source_input.lcm_maintenance_debt s + WHERE NOT EXISTS ( + SELECT 1 FROM target_input.lcm_maintenance_debt t + WHERE t.provider=s.provider AND t.conversation_id=s.conversation_id + AND t.debt_id=s.debt_id + )" + ), + )); + + specs.extend([ + newest_wins( + "workflow run", + "workflow_runs", + "run_id, parent_session_id, name, description, phase_json, status, started_ts, ended_ts, result_summary, agent_count, created_at, updated_at", + "t.run_id=s.run_id", + ), + newest_wins( + "workflow agent", + "workflow_agents", + "run_id, agent_label, agent_id, phase, transcript_path, agent_session_id, status, model, tokens, started_ts, ended_ts, created_at, updated_at", + "t.run_id=s.run_id AND t.agent_label=s.agent_label AND t.agent_id=s.agent_id", + ), + max_meta("workflow index metadata", "workflow_index_meta"), + offset_append( + "session git span", + "session_git_spans", + "span_id, provider, session_id, thread_id, branch, worktree, first_ts, last_ts, event_count, source, created_at, updated_at", + &format!( + "span_id + {}, provider, session_id, thread_id, branch, worktree, first_ts, last_ts, event_count, source, created_at, updated_at", + offsets.span + ), + ), + commit_sessions(offsets.span), + max_meta("git correlation metadata", "git_correlation_meta"), + max_meta("session backfill metadata", "session_backfill_meta"), + target_wins( + "dashboard token count", + "dashboard_token_counts", + "store, provider, message_id, text_len, encoder, token_count, computed_at", + "t.store=s.store AND t.provider=s.provider AND t.message_id=s.message_id", + ), + ]); + specs +} + +fn target_wins( + label: &'static str, + table: &'static str, + columns: &'static str, + key: &'static str, +) -> TableVerification { + projected_target_wins(label, table, columns, columns, key) +} + +fn projected_target_wins( + label: &'static str, + table: &'static str, + columns: &'static str, + source_projection: &str, + key: &'static str, +) -> TableVerification { + custom_owned( + label, + table, + columns, + format!( + "SELECT {columns} FROM target_input.{table} + UNION ALL + SELECT {source_projection} FROM source_input.{table} s + WHERE NOT EXISTS (SELECT 1 FROM target_input.{table} t WHERE {key})" + ), + ) +} + +fn offset_append( + label: &'static str, + table: &'static str, + columns: &'static str, + source_projection: &str, +) -> TableVerification { + custom_owned( + label, + table, + columns, + format!( + "SELECT {columns} FROM target_input.{table} + UNION ALL SELECT {source_projection} FROM source_input.{table}" + ), + ) +} + +fn newest_wins( + label: &'static str, + table: &'static str, + columns: &'static str, + key: &'static str, +) -> TableVerification { + let target_columns = prefixed_columns(columns, "t"); + let source_columns = prefixed_columns(columns, "s"); + custom_owned( + label, + table, + columns, + format!( + "SELECT {target_columns} FROM target_input.{table} t + WHERE NOT EXISTS ( + SELECT 1 FROM source_input.{table} s + WHERE {key} AND s.updated_at > t.updated_at + ) + UNION ALL + SELECT {source_columns} FROM source_input.{table} s + WHERE NOT EXISTS ( + SELECT 1 FROM target_input.{table} t + WHERE {key} AND t.updated_at >= s.updated_at + )" + ), + ) +} + +fn max_meta(label: &'static str, table: &'static str) -> TableVerification { + custom_owned( + label, + table, + "key, value, updated_at", + format!( + "SELECT key, MAX(value), MAX(updated_at) FROM ( + SELECT key, value, updated_at FROM target_input.{table} + UNION ALL SELECT key, value, updated_at FROM source_input.{table} + ) GROUP BY key" + ), + ) +} + +fn commit_sessions(span_offset: i64) -> TableVerification { + custom_owned( + "commit-session attribution", + "commit_sessions", + "commit_sha, provider, session_id, branch, worktree, committed_at, span_overlap_kind, span_id, relation, evidence, confidence, evidence_message_id, created_at", + format!( + "SELECT t.commit_sha, t.provider, t.session_id, + CASE WHEN s.confidence > t.confidence THEN s.branch ELSE t.branch END, + CASE WHEN s.confidence > t.confidence THEN s.worktree ELSE t.worktree END, + CASE WHEN s.confidence > t.confidence THEN s.committed_at ELSE t.committed_at END, + CASE WHEN s.confidence > t.confidence THEN s.span_overlap_kind ELSE t.span_overlap_kind END, + CASE WHEN s.confidence > t.confidence + THEN CASE WHEN s.span_id IS NULL THEN NULL ELSE s.span_id + {span_offset} END + ELSE t.span_id END, + CASE WHEN s.confidence > t.confidence THEN s.relation ELSE t.relation END, + CASE WHEN s.confidence > t.confidence THEN s.evidence ELSE t.evidence END, + MAX(t.confidence, s.confidence), + CASE WHEN s.confidence > t.confidence THEN s.evidence_message_id ELSE t.evidence_message_id END, + t.created_at + FROM target_input.commit_sessions t + JOIN source_input.commit_sessions s + ON s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id + UNION ALL + SELECT t.commit_sha, t.provider, t.session_id, t.branch, t.worktree, + t.committed_at, t.span_overlap_kind, t.span_id, t.relation, + t.evidence, t.confidence, t.evidence_message_id, t.created_at + FROM target_input.commit_sessions t + WHERE NOT EXISTS ( + SELECT 1 FROM source_input.commit_sessions s + WHERE s.commit_sha=t.commit_sha AND s.provider=t.provider AND s.session_id=t.session_id + ) + UNION ALL + SELECT s.commit_sha, s.provider, s.session_id, s.branch, s.worktree, + s.committed_at, s.span_overlap_kind, + CASE WHEN s.span_id IS NULL THEN NULL ELSE s.span_id + {span_offset} END, + s.relation, s.evidence, s.confidence, s.evidence_message_id, s.created_at + FROM source_input.commit_sessions s + WHERE NOT EXISTS ( + SELECT 1 FROM target_input.commit_sessions t + WHERE t.commit_sha=s.commit_sha AND t.provider=s.provider AND t.session_id=s.session_id + )" + ), + ) +} + +fn remapped_raw_id(expression: &str, offset: i64) -> String { + format!( + "(SELECT COALESCE(t.store_id, r.store_id + {offset}) + FROM source_input.lcm_raw_messages r + LEFT JOIN target_input.lcm_raw_messages t + ON t.provider=r.provider AND t.message_id=r.message_id + WHERE r.store_id={expression})" + ) +} + +fn prefixed_columns(columns: &str, alias: &str) -> String { + columns + .split(',') + .map(|column| format!("{alias}.{}", column.trim())) + .collect::>() + .join(", ") +} + +fn custom( + label: &'static str, + table: &'static str, + columns: &'static str, + expected: &'static str, +) -> TableVerification { + custom_owned(label, table, columns, expected.to_string()) +} + +fn custom_owned( + label: &'static str, + table: &'static str, + columns: &'static str, + expected: String, +) -> TableVerification { + TableVerification { + label, + table, + columns, + expected, + } +} + +async fn verify_payload_files(conn: &Connection, destination_root: &Path) -> Result<()> { + let mut rows = conn + .query( + "SELECT payload_ref, content_hash, byte_count FROM lcm_external_payloads", + (), + ) + .await + .map_err(|error| db_error("verify_consolidation", error))?; + while let Some(row) = rows + .next() + .await + .map_err(|error| db_error("verify_consolidation", error))? + { + let payload_ref = row + .get::(0) + .map_err(|error| db_error("verify_consolidation", error))?; + let content_hash = row + .get::(1) + .map_err(|error| db_error("verify_consolidation", error))?; + let byte_count = row + .get::(2) + .map_err(|error| db_error("verify_consolidation", error))?; + crate::sessions::lcm::payload::validate_payload_ref(&payload_ref).map_err(|_| { + db_message( + "verify_consolidation", + format!("destination contains invalid external payload ref '{payload_ref}'"), + ) + })?; + let path = destination_root.join("lcm-payloads").join(&payload_ref); + let metadata = std::fs::symlink_metadata(&path).map_err(|error| { + db_message( + "verify_consolidation", + format!( + "destination external payload '{}' is missing: {error}", + path.display() + ), + ) + })?; + if !metadata.is_file() || i64::try_from(metadata.len()).ok() != Some(byte_count) { + return Err(db_message( + "verify_consolidation", + format!( + "destination external payload '{}' has the wrong file shape or length", + path.display() + ), + )); + } + let digest = super::super::files::file_digest(&path)?; + if hex::encode(digest) != content_hash { + return Err(db_message( + "verify_consolidation", + format!( + "destination external payload '{}' failed content hash verification", + path.display() + ), + )); + } + } + Ok(()) +} diff --git a/src/migrate/consolidate/tests.rs b/src/migrate/consolidate/tests.rs new file mode 100644 index 00000000..a78e5fa7 --- /dev/null +++ b/src/migrate/consolidate/tests.rs @@ -0,0 +1,1199 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::SystemTime; + +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; + +use serde_json::json; +use tempfile::TempDir; + +use super::*; +use crate::db::Database; +use crate::memory::store::MemoryStore; +use crate::memory::types::{AddFactRequest, FeedbackAction, FeedbackRequest, MemoryCategory}; +use crate::sessions::{SessionMessageRecord, SessionRecord}; +use crate::tracedecay::{TraceDecay, TraceDecayOpenOptions}; + +struct Fixture { + _temp: TempDir, + project: PathBuf, + profile: PathBuf, + source_id: String, + target_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum SnapshotEntry { + Missing, + File { + digest: [u8; 32], + bytes: u64, + modified: SystemTime, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, + #[cfg(unix)] + changed_seconds: i64, + #[cfg(unix)] + changed_nanoseconds: i64, + #[cfg(unix)] + links: u64, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum TreeSnapshotEntry { + Directory { + modified: SystemTime, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, + #[cfg(unix)] + changed_seconds: i64, + #[cfg(unix)] + changed_nanoseconds: i64, + #[cfg(unix)] + mode: u32, + }, + File(SnapshotEntry), +} + +fn migration_surface_snapshot(fixture: &Fixture) -> BTreeMap { + let mut snapshot = BTreeMap::new(); + for root in [ + fixture.profile.join("projects").join(&fixture.source_id), + fixture.profile.join("projects").join(&fixture.target_id), + ] { + for path in relative_file_map(&root).unwrap().into_values() { + snapshot_file(&path, &mut snapshot); + } + } + let global = fixture.profile.join("global.db"); + for path in [ + storage::enrollment_marker_path(&fixture.project), + storage::repository_identity_path(&fixture.project).unwrap(), + global.clone(), + sqlite_sidecar(&global, "-wal"), + sqlite_sidecar(&global, "-shm"), + ] { + snapshot_file(&path, &mut snapshot); + } + snapshot +} + +fn snapshot_file(path: &Path, snapshot: &mut BTreeMap) { + let entry = if path.is_file() { + let metadata = fs::metadata(path).unwrap(); + SnapshotEntry::File { + digest: file_digest(path).unwrap(), + bytes: metadata.len(), + modified: metadata.modified().unwrap(), + #[cfg(unix)] + device: metadata.dev(), + #[cfg(unix)] + inode: metadata.ino(), + #[cfg(unix)] + changed_seconds: metadata.ctime(), + #[cfg(unix)] + changed_nanoseconds: metadata.ctime_nsec(), + #[cfg(unix)] + links: metadata.nlink(), + } + } else { + SnapshotEntry::Missing + }; + snapshot.insert(path.to_path_buf(), entry); +} + +fn full_tree_snapshot(root: &Path) -> BTreeMap { + let mut snapshot = BTreeMap::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(path) = pending.pop() { + let metadata = fs::symlink_metadata(&path).unwrap(); + let relative = path.strip_prefix(root).unwrap().to_path_buf(); + if metadata.is_dir() { + #[cfg(unix)] + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + snapshot.insert( + relative, + TreeSnapshotEntry::Directory { + modified: metadata.modified().unwrap(), + #[cfg(unix)] + device: metadata.dev(), + #[cfg(unix)] + inode: metadata.ino(), + #[cfg(unix)] + changed_seconds: metadata.ctime(), + #[cfg(unix)] + changed_nanoseconds: metadata.ctime_nsec(), + #[cfg(unix)] + mode: metadata.permissions().mode(), + }, + ); + let mut children = fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::>(); + children.sort(); + pending.extend(children); + } else { + let mut file = BTreeMap::new(); + snapshot_file(&path, &mut file); + snapshot.insert( + relative, + TreeSnapshotEntry::File(file.remove(&path).unwrap()), + ); + } + } + snapshot +} + +impl Fixture { + fn options(&self) -> ConsolidationOptions { + ConsolidationOptions { + project_root: self.project.clone(), + profile_root: self.profile.clone(), + source_project_id: self.source_id.clone(), + target_project_id: self.target_id.clone(), + } + } +} + +#[tokio::test] +async fn dry_run_reports_live_split_shape_without_mutation() { + let fixture = fixture().await; + let before = migration_surface_snapshot(&fixture); + let profile_before = full_tree_snapshot(&fixture.profile); + assert!(!fixture.profile.join("lifecycle.lock").exists()); + let report = plan(&fixture.options()).await.unwrap(); + let after = migration_surface_snapshot(&fixture); + let profile_after = full_tree_snapshot(&fixture.profile); + + assert!(report.dry_run); + assert_eq!(report.state, ConsolidationState::Planned); + assert_eq!(report.source.facts, 1); + assert_eq!(report.source.feedback_events, 1); + assert_eq!(report.target.facts, 1); + assert_eq!(report.source.sessions, 1); + assert_eq!(report.target.sessions, 1); + assert_eq!(report.source.lcm_raw_messages, 1); + assert_eq!(report.target.lcm_raw_messages, 1); + assert!(!report.destination_data_root.exists()); + assert!(!report.backup_root.exists()); + assert!(!report.ledger_path.exists()); + assert_eq!(after, before, "dry-run changed an input or identity file"); + assert_eq!( + profile_after, profile_before, + "dry-run changed the profile tree" + ); + assert!( + fs::read_dir(fixture.profile.parent().unwrap()) + .unwrap() + .all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".tracedecay-migration-scratch-")), + "dry-run left migration scratch state behind" + ); + assert!(!fixture.profile.join("lifecycle.lock").exists()); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + fixture.target_id + ); +} + +#[tokio::test] +async fn many_branch_plan_retains_constant_database_handles_and_bounded_scratch() { + const BRANCHES_PER_SHARD: usize = 48; + let fixture = fixture().await; + add_branch_links(&fixture, &fixture.source_id, BRANCHES_PER_SHARD); + add_branch_links(&fixture, &fixture.target_id, BRANCHES_PER_SHARD); + + let resolved = resolve_plan(&fixture.options()).await.unwrap(); + assert_eq!( + resolved.report.source.graph_databases, + BRANCHES_PER_SHARD + 1 + ); + assert_eq!( + resolved.report.target.graph_databases, + BRANCHES_PER_SHARD + 1 + ); + assert_eq!( + resolved.evidence.retained_database_count(), + 2, + "graph snapshots must be processed and dropped one at a time; only the two session snapshots stay open" + ); + + let max_graph_family = input_database_paths(&resolved) + .unwrap() + .into_iter() + .filter(|path| { + path.file_name().and_then(|name| name.to_str()) != Some(storage::SESSIONS_DB_FILENAME) + }) + .map(|path| sqlite_family_bytes(&path)) + .max() + .unwrap(); + assert!( + resolved.evidence.peak_graph_scratch_bytes() <= max_graph_family, + "graph scratch must be bounded by one SQLite family, independent of branch count" + ); + let session_family_bytes = [ + &resolved.source_layout.sessions_db_path, + &resolved.target_layout.sessions_db_path, + ] + .into_iter() + .map(|path| sqlite_family_bytes(path)) + .sum::(); + assert!( + resolved.evidence.sessions.copied_bytes() <= session_family_bytes, + "retained session scratch must be bounded by the two input families" + ); +} + +#[tokio::test] +async fn interrupted_apply_retries_without_duplicates_and_cuts_over_last() { + let fixture = fixture().await; + let options = fixture.options(); + let source_root = fixture.profile.join("projects").join(&fixture.source_id); + let target_root = fixture.profile.join("projects").join(&fixture.target_id); + fs::write(source_root.join(".dirty"), b"interrupted source sync").unwrap(); + fs::write( + source_root.join("tracedecay.db.corrupt-enospc-source"), + b"source forensic database", + ) + .unwrap(); + fs::write( + target_root.join("tracedecay.db.corrupt-enospc-target"), + b"target forensic database", + ) + .unwrap(); + let report = plan(&options).await.unwrap(); + + let error = apply_with_stop( + &options, + &report.confirmation_token, + Some(ConsolidationState::DatabasesMerged), + ) + .await + .unwrap_err(); + assert!( + error.to_string().contains("synthetic interruption"), + "{error}" + ); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + fixture.target_id, + "marker must not move before all data and registry phases succeed" + ); + + let applied = apply(&options, &report.confirmation_token).await.unwrap(); + assert_eq!(applied.state, ConsolidationState::Applied); + assert!(!applied.dry_run); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + applied.destination_project_id + ); + assert_eq!( + storage::read_enrollment_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + applied.destination_project_id, + "successful cutover must suppress legacy shard discovery even when no enrollment marker existed" + ); + + let graph = applied + .destination_data_root + .join(crate::config::DB_FILENAME); + let sessions = applied + .destination_data_root + .join(storage::SESSIONS_DB_FILENAME); + assert_eq!(sqlite::count_rows(&graph, "memory_facts").await.unwrap(), 2); + assert_eq!( + sqlite::count_rows(&graph, "memory_feedback_events") + .await + .unwrap(), + 1 + ); + assert_eq!(sqlite::count_rows(&sessions, "sessions").await.unwrap(), 2); + assert_eq!( + sqlite::count_rows(&sessions, "session_messages") + .await + .unwrap(), + 2 + ); + assert_eq!( + sqlite::count_rows(&sessions, "lcm_raw_messages") + .await + .unwrap(), + 2 + ); + assert!( + applied + .destination_data_root + .join("lcm-payloads/source.txt") + .is_file() + ); + assert!( + applied + .destination_data_root + .join("lcm-payloads/target.txt") + .is_file() + ); + assert!( + applied + .backup_root + .join(&fixture.source_id) + .join(storage::STORE_MANIFEST_FILENAME) + .is_file() + ); + assert_eq!( + fs::read(applied.backup_root.join(&fixture.source_id).join(".dirty")).unwrap(), + b"interrupted source sync" + ); + assert_eq!( + fs::read( + applied + .destination_data_root + .join("tracedecay.db.corrupt-enospc-source") + ) + .unwrap(), + b"source forensic database" + ); + assert_eq!( + fs::read( + applied + .destination_data_root + .join("tracedecay.db.corrupt-enospc-target") + ) + .unwrap(), + b"target forensic database" + ); + assert!( + applied + .backup_root + .join(&fixture.target_id) + .join(storage::SESSIONS_DB_FILENAME) + .is_file() + ); + assert!( + fixture + .profile + .join("projects") + .join(&fixture.source_id) + .is_dir() + ); + assert!( + fixture + .profile + .join("projects") + .join(&fixture.target_id) + .is_dir() + ); + + let meta = branch_meta::load_branch_meta(&applied.destination_data_root).unwrap(); + assert!(meta.branches.contains_key("main")); + assert!( + meta.branches + .contains_key(&format!("consolidated/{}/main", fixture.source_id)) + ); + + let retried = apply(&options, &report.confirmation_token).await.unwrap(); + assert_eq!(retried.state, ConsolidationState::Applied); + assert_eq!(sqlite::count_rows(&graph, "memory_facts").await.unwrap(), 2); + assert_eq!(sqlite::count_rows(&sessions, "sessions").await.unwrap(), 2); +} + +#[tokio::test] +async fn destination_preparation_restarts_after_every_publish_boundary() { + for stop in [ + prepare::PrepareStop::TargetCopy, + prepare::PrepareStop::SourceBranch(1), + prepare::PrepareStop::BranchMetaWrite, + prepare::PrepareStop::Publish, + ] { + let fixture = fixture().await; + let options = fixture.options(); + let report = plan(&options).await.unwrap(); + + let error = apply_with_prepare_stop(&options, &report.confirmation_token, stop) + .await + .unwrap_err(); + assert!( + error.to_string().contains("synthetic interruption"), + "{stop:?}: {error}" + ); + + let applied = apply(&options, &report.confirmation_token).await.unwrap(); + assert_eq!(applied.state, ConsolidationState::Applied, "{stop:?}"); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + applied.destination_project_id, + "{stop:?}" + ); + } +} + +#[tokio::test] +async fn consolidation_restarts_after_every_durable_state() { + for stop in [ + ConsolidationState::BackupsReady, + ConsolidationState::DestinationReady, + ConsolidationState::DatabasesMerged, + ConsolidationState::ArtifactsMerged, + ConsolidationState::Registered, + ] { + let fixture = fixture().await; + let options = fixture.options(); + let report = plan(&options).await.unwrap(); + + let error = apply_with_stop(&options, &report.confirmation_token, Some(stop.clone())) + .await + .unwrap_err(); + assert!( + error.to_string().contains("synthetic interruption"), + "{stop:?}: {error}" + ); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + fixture.target_id, + "{stop:?}: marker moved before the final state" + ); + + let applied = apply(&options, &report.confirmation_token).await.unwrap(); + assert_eq!(applied.state, ConsolidationState::Applied, "{stop:?}"); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + applied.destination_project_id, + "{stop:?}" + ); + } +} + +#[tokio::test] +async fn verification_rejects_a_missing_unique_row_when_target_is_larger() { + let fixture = fixture().await; + for suffix in ["one", "two"] { + add_fact_to_shard( + &fixture, + &fixture.target_id, + &format!("extra target fact {suffix}"), + "target-extra", + json!({"suffix": suffix}), + None, + ) + .await; + } + let options = fixture.options(); + let report = plan(&options).await.unwrap(); + apply_with_stop( + &options, + &report.confirmation_token, + Some(ConsolidationState::DatabasesMerged), + ) + .await + .unwrap_err(); + + let graph_path = report + .destination_data_root + .join(crate::config::DB_FILENAME); + let (graph, _) = Database::open(&graph_path).await.unwrap(); + graph + .conn() + .execute_batch( + "PRAGMA foreign_keys = OFF; + DELETE FROM memory_facts WHERE content = 'legacy durable fact';", + ) + .await + .unwrap(); + graph.checkpoint().await.unwrap(); + graph.close(); + assert_eq!( + sqlite::count_rows(&graph_path, "memory_facts") + .await + .unwrap(), + report.target.facts, + "the old max(input) count check would have accepted this loss" + ); + + let error = apply(&options, &report.confirmation_token) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("destination fact logical union differs"), + "{error}" + ); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + fixture.target_id + ); +} + +#[tokio::test] +async fn verification_checks_session_bounds_and_immutable_message_payloads() { + let fixture = fixture().await; + let options = fixture.options(); + let report = plan(&options).await.unwrap(); + apply_with_stop( + &options, + &report.confirmation_token, + Some(ConsolidationState::DatabasesMerged), + ) + .await + .unwrap_err(); + let sessions = report + .destination_data_root + .join(storage::SESSIONS_DB_FILENAME); + + execute_sql( + &sessions, + "UPDATE sessions SET ended_at=42 WHERE session_id='legacy-session'", + ) + .await; + let error = apply(&options, &report.confirmation_token) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("destination session logical union differs"), + "{error}" + ); + + execute_sql( + &sessions, + "UPDATE sessions SET ended_at=1800000001 WHERE session_id='legacy-session'; + UPDATE session_messages SET text='corrupted text' + WHERE message_id='message-legacy-session';", + ) + .await; + let error = apply(&options, &report.confirmation_token) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("destination session message logical union differs"), + "{error}" + ); + + execute_sql( + &sessions, + "UPDATE session_messages SET text='message from legacy-session' + WHERE message_id='message-legacy-session'; + UPDATE lcm_raw_messages SET content_hash='corrupted-hash' + WHERE session_id='legacy-session';", + ) + .await; + let error = apply(&options, &report.confirmation_token) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("destination LCM raw message logical union differs"), + "{error}" + ); +} + +#[tokio::test] +async fn identity_survives_symlink_and_repository_move() { + let fixture = fixture().await; + let symlink = fixture.project.parent().unwrap().join("repo-symlink"); + #[cfg(unix)] + std::os::unix::fs::symlink(&fixture.project, &symlink).unwrap(); + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&fixture.project, &symlink).unwrap(); + let mut options = fixture.options(); + options.project_root = symlink; + let report = plan(&options).await.unwrap(); + let applied = apply(&options, &report.confirmation_token).await.unwrap(); + assert_eq!( + storage::read_enrollment_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + applied.destination_project_id + ); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + applied.destination_project_id + ); + + let moved = fixture.project.parent().unwrap().join("repo-moved"); + fs::rename(&fixture.project, &moved).unwrap(); + let reopened = TraceDecay::open_read_only_with_options( + &moved, + TraceDecayOpenOptions { + profile_root: Some(fixture.profile.clone()), + global_db_path: Some(fixture.profile.join("global.db")), + }, + ) + .await + .unwrap(); + assert!(same_path( + &reopened.store_layout().data_root, + &applied.destination_data_root + )); +} + +#[tokio::test] +async fn a_third_matching_shard_is_rejected_as_ambiguous() { + let fixture = fixture().await; + create_shard( + &fixture.profile, + &fixture.project, + "proj_third", + "third fact", + "third-session", + false, + ) + .await; + let error = plan(&fixture.options()).await.unwrap_err(); + assert!(error.to_string().contains("ambiguous split-store identity")); + assert!(error.to_string().contains("proj_third")); +} + +#[tokio::test] +async fn overlapping_facts_merge_tags_metadata_and_feedback_without_duplication() { + let fixture = fixture().await; + add_fact_to_shard( + &fixture, + &fixture.source_id, + "shared fact", + "source-tag", + json!({"source_only": true, "winner": "source"}), + Some(FeedbackAction::Helpful), + ) + .await; + add_fact_to_shard( + &fixture, + &fixture.target_id, + "shared fact", + "target-tag", + json!({"target_only": true, "winner": "target"}), + Some(FeedbackAction::Unhelpful), + ) + .await; + + let options = fixture.options(); + let planned = plan(&options).await.unwrap(); + assert_eq!(planned.collisions.fact_content_overlaps, 1); + let applied = apply(&options, &planned.confirmation_token).await.unwrap(); + let graph_path = applied + .destination_data_root + .join(crate::config::DB_FILENAME); + let (graph, _) = Database::open_read_only(&graph_path).await.unwrap(); + let store = MemoryStore::new(graph.conn()); + let facts = store.list_facts(None, Some(0.0), 100).await.unwrap(); + let shared = facts + .iter() + .find(|fact| fact.content == "shared fact") + .unwrap(); + assert_eq!(facts.len(), 3); + assert!(shared.tags.contains(&"source-tag".to_string())); + assert!(shared.tags.contains(&"target-tag".to_string())); + assert_eq!(shared.metadata["source_only"], true); + assert_eq!(shared.metadata["target_only"], true); + assert_eq!(shared.metadata["winner"], "target"); + assert_eq!(shared.helpful_count, 1); + assert_eq!(shared.unhelpful_count, 1); + assert_eq!( + store + .fact_trust_history(shared.fact_id) + .await + .unwrap() + .len(), + 2 + ); + graph.close(); +} + +#[tokio::test] +async fn summary_raw_sources_follow_remapped_store_ids() { + let fixture = fixture().await; + let source = layout_for_id(&fixture.project, &fixture.profile, &fixture.source_id).unwrap(); + execute_sql( + &source.sessions_db_path, + "INSERT INTO lcm_summary_nodes( + node_id, provider, conversation_id, session_id, depth, summary_text, + summary_hash, summary_token_count, source_token_count, created_at + ) VALUES( + 'source-summary', 'codex', 'source-conversation', 'legacy-session', 1, + 'summary', 'summary-hash', 1, 1, 1800000002 + ); + INSERT INTO lcm_summary_sources(node_id, source_kind, source_id, ordinal) + SELECT 'source-summary', 'raw_message', CAST(store_id AS TEXT), 0 + FROM lcm_raw_messages WHERE message_id='message-legacy-session';", + ) + .await; + + let options = fixture.options(); + let planned = plan(&options).await.unwrap(); + let applied = apply(&options, &planned.confirmation_token).await.unwrap(); + let sessions = GlobalDb::open_at( + &applied + .destination_data_root + .join(storage::SESSIONS_DB_FILENAME), + ) + .await + .unwrap(); + let mut rows = sessions + .conn() + .query( + "SELECT r.message_id + FROM lcm_summary_sources s + JOIN lcm_raw_messages r ON r.store_id=CAST(s.source_id AS INTEGER) + WHERE s.node_id='source-summary' AND s.source_kind='raw_message'", + (), + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().unwrap(); + assert_eq!(row.get::(0).unwrap(), "message-legacy-session"); + assert!(rows.next().await.unwrap().is_none()); + drop(rows); + sessions.close(); +} + +#[tokio::test] +async fn preexisting_destination_without_ledger_is_never_reused() { + let fixture = fixture().await; + let options = fixture.options(); + let planned = plan(&options).await.unwrap(); + fs::create_dir_all(&planned.destination_data_root).unwrap(); + fs::write(planned.destination_data_root.join("foreign"), b"foreign").unwrap(); + + let error = apply(&options, &planned.confirmation_token) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("already exists without this migration ledger") + ); + assert!(!planned.ledger_path.exists()); + assert_eq!( + storage::read_repository_identity_marker(&fixture.project) + .unwrap() + .unwrap() + .project_id, + fixture.target_id + ); +} + +#[tokio::test] +async fn corrupt_retry_ledger_is_never_overwritten() { + let fixture = fixture().await; + let options = fixture.options(); + let planned = plan(&options).await.unwrap(); + fs::create_dir_all(planned.ledger_path.parent().unwrap()).unwrap(); + fs::write(&planned.ledger_path, b"{not-json").unwrap(); + + let error = apply(&options, &planned.confirmation_token) + .await + .unwrap_err(); + assert!(error.to_string().contains("ledger")); + assert!(error.to_string().contains("corrupt")); + assert_eq!(fs::read(&planned.ledger_path).unwrap(), b"{not-json"); + assert!(!planned.destination_data_root.exists()); +} + +#[test] +fn sqlite_family_backup_includes_wal_and_shm() { + let temp = TempDir::new().unwrap(); + let source = temp.path().join("source.db"); + let target = temp.path().join("backup/target.db"); + fs::write(&source, b"db").unwrap(); + fs::write(sqlite_sidecar(&source, "-wal"), b"wal").unwrap(); + fs::write(sqlite_sidecar(&source, "-shm"), b"shm").unwrap(); + + copy_sqlite_family_exact(&source, &target).unwrap(); + + assert_eq!(fs::read(&target).unwrap(), b"db"); + assert_eq!(fs::read(sqlite_sidecar(&target, "-wal")).unwrap(), b"wal"); + assert_eq!(fs::read(sqlite_sidecar(&target, "-shm")).unwrap(), b"shm"); +} + +#[test] +fn atomic_copy_recovers_an_interrupted_temp_and_reopens_cleanly() { + let temp = TempDir::new().unwrap(); + let source = temp.path().join("source.bin"); + let target = temp.path().join("backup/target.bin"); + let interrupted = target.with_extension(format!("tmp-{}", std::process::id())); + fs::create_dir_all(interrupted.parent().unwrap()).unwrap(); + fs::write(&source, b"durable source bytes").unwrap(); + fs::write(&interrupted, b"partial").unwrap(); + + copy_file_atomic(&source, &target).unwrap(); + + assert_eq!(fs::read(&target).unwrap(), b"durable source bytes"); + assert!(!interrupted.exists()); +} + +#[tokio::test] +async fn current_schema_tables_have_an_explicit_consolidation_disposition() { + let fixture = fixture().await; + let graph = fixture + .profile + .join("projects") + .join(&fixture.source_id) + .join(crate::config::DB_FILENAME); + let sessions = fixture + .profile + .join("projects") + .join(&fixture.source_id) + .join(storage::SESSIONS_DB_FILENAME); + + let unknown_graph = unknown_tables(&graph, graph_table_disposition).await; + let unknown_sessions = unknown_tables(&sessions, session_table_disposition).await; + + assert!( + unknown_graph.is_empty(), + "graph schema tables need an explicit consolidation disposition: {unknown_graph:?}" + ); + assert!( + unknown_sessions.is_empty(), + "session schema tables need an explicit consolidation disposition: {unknown_sessions:?}" + ); +} + +async fn unknown_tables(path: &Path, classify: fn(&str) -> Option<&'static str>) -> Vec { + let (db, _) = Database::open_read_only(path).await.unwrap(); + let mut rows = db + .conn() + .query( + "SELECT name FROM sqlite_schema + WHERE type='table' AND name NOT LIKE 'sqlite_%' + ORDER BY name", + (), + ) + .await + .unwrap(); + let mut unknown = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + let name = row.get::(0).unwrap(); + if classify(&name).is_none() { + unknown.push(name); + } + } + db.close(); + unknown +} + +fn graph_table_disposition(table: &str) -> Option<&'static str> { + match table { + "memory_entities" + | "memory_fact_entities" + | "memory_facts" + | "memory_feedback_events" + | "memory_oplog" => Some("merged"), + "memory_bank_dirty" | "memory_banks" => Some("derived/rebuilt"), + name if name == "memory_facts_fts" || name.starts_with("memory_facts_fts_") => { + Some("derived/rebuilt") + } + // Code-graph tables are not flattened. Every source and target branch + // database is copied intact into the destination branch topology. + "edges" | "files" | "metadata" | "node_fingerprints" | "nodes" | "read_cache" + | "redundancy_pairs" | "unresolved_refs" | "vectors" => Some("intentionally ignored"), + name if name == "nodes_fts" || name.starts_with("nodes_fts_") => { + Some("intentionally ignored") + } + _ => None, + } +} + +fn session_table_disposition(table: &str) -> Option<&'static str> { + match table { + "analytics_events" + | "commit_sessions" + | "dashboard_token_counts" + | "git_correlation_meta" + | "lcm_external_payloads" + | "lcm_gc_marks" + | "lcm_gc_meta" + | "lcm_lifecycle_state" + | "lcm_maintenance_debt" + | "lcm_raw_messages" + | "lcm_summary_nodes" + | "lcm_summary_sources" + | "parse_offsets" + | "projects" + | "savings_ledger" + | "session_backfill_meta" + | "session_git_spans" + | "session_messages" + | "session_schema_migrations" + | "sessions" + | "turns" + | "workflow_agents" + | "workflow_index_meta" + | "workflow_runs" => Some("merged"), + "code_projects" | "graph_scopes" | "project_aliases" | "store_artifacts" + | "store_instances" => Some("rejected registry-only"), + name if name == "lcm_raw_messages_fts" + || name.starts_with("lcm_raw_messages_fts_") + || name == "lcm_summary_nodes_fts" + || name.starts_with("lcm_summary_nodes_fts_") + || name == "session_messages_fts" + || name.starts_with("session_messages_fts_") => + { + Some("derived/rebuilt") + } + _ => None, + } +} + +async fn fixture() -> Fixture { + let temp = TempDir::new().unwrap(); + let project = temp.path().join("repo"); + let profile = temp.path().join("profile"); + let source_id = "proj_legacy".to_string(); + let target_id = "proj_current".to_string(); + init_repo(&project); + create_shard( + &profile, + &project, + &source_id, + "legacy durable fact", + "legacy-session", + true, + ) + .await; + create_shard( + &profile, + &project, + &target_id, + "current durable fact", + "current-session", + false, + ) + .await; + storage::write_repository_identity_marker(&project, &target_id).unwrap(); + Fixture { + _temp: temp, + project, + profile, + source_id, + target_id, + } +} + +async fn create_shard( + profile: &Path, + project: &Path, + project_id: &str, + fact_content: &str, + session_id: &str, + feedback: bool, +) { + let layout = layout_for_id(project, profile, project_id).unwrap(); + fs::create_dir_all(&layout.data_root).unwrap(); + let (graph, _) = Database::initialize(&layout.graph_db_path).await.unwrap(); + let memory = MemoryStore::new(graph.conn()); + let outcome = memory + .add_fact( + AddFactRequest { + content: fact_content.to_string(), + category: MemoryCategory::Project, + source: Some("consolidation-test".to_string()), + tags: vec![project_id.to_string()], + entities: vec!["TraceDecay".to_string()], + trust: Some(0.8), + metadata: json!({"project_id": project_id}), + }, + 0.5, + ) + .await + .unwrap(); + if feedback { + memory + .record_feedback_event(FeedbackRequest { + fact_id: outcome.fact.unwrap().fact_id, + action: FeedbackAction::Helpful, + source: Some("consolidation-test".to_string()), + note: Some("verified".to_string()), + }) + .await + .unwrap(); + } + graph.checkpoint().await.unwrap(); + graph.close(); + + let sessions = GlobalDb::open_at(&layout.sessions_db_path).await.unwrap(); + assert!( + sessions + .upsert_session(&SessionRecord { + provider: "codex".to_string(), + session_id: session_id.to_string(), + project_key: project_id.to_string(), + project_path: project.to_string_lossy().to_string(), + title: Some(session_id.to_string()), + started_at: Some(1_800_000_000), + ended_at: Some(1_800_000_001), + transcript_path: None, + metadata_json: None, + parent_session_id: None, + is_subagent: false, + agent_id: None, + parent_tool_use_id: None, + }) + .await + ); + assert!( + sessions + .upsert_session_message(&SessionMessageRecord { + provider: "codex".to_string(), + message_id: format!("message-{session_id}"), + session_id: session_id.to_string(), + role: "user".to_string(), + timestamp: Some(1_800_000_000), + ordinal: 0, + text: format!("message from {session_id}"), + kind: Some("message".to_string()), + model: None, + tool_names: None, + source_path: None, + source_offset: None, + metadata_json: None, + }) + .await + ); + sessions.checkpoint().await; + sessions.close(); + + branch_meta::save_branch_meta(&layout.data_root, &BranchMeta::new("main")).unwrap(); + fs::create_dir_all(layout.data_root.join("lcm-payloads")).unwrap(); + let payload_name = if feedback { "source.txt" } else { "target.txt" }; + fs::write( + layout.data_root.join("lcm-payloads").join(payload_name), + session_id, + ) + .unwrap(); + storage::write_store_manifest(&layout).unwrap(); +} + +async fn add_fact_to_shard( + fixture: &Fixture, + project_id: &str, + content: &str, + tag: &str, + metadata: serde_json::Value, + feedback: Option, +) { + let layout = layout_for_id(&fixture.project, &fixture.profile, project_id).unwrap(); + let (graph, _) = Database::open(&layout.graph_db_path).await.unwrap(); + let memory = MemoryStore::new(graph.conn()); + let outcome = memory + .add_fact( + AddFactRequest { + content: content.to_string(), + category: MemoryCategory::Project, + source: Some(project_id.to_string()), + tags: vec![tag.to_string()], + entities: vec!["TraceDecay".to_string()], + trust: Some(0.8), + metadata, + }, + 0.5, + ) + .await + .unwrap(); + if let Some(action) = feedback { + memory + .record_feedback_event(FeedbackRequest { + fact_id: outcome.fact.unwrap().fact_id, + action, + source: Some(project_id.to_string()), + note: Some("overlap".to_string()), + }) + .await + .unwrap(); + } + graph.checkpoint().await.unwrap(); + graph.close(); +} + +fn add_branch_links(fixture: &Fixture, project_id: &str, count: usize) { + let layout = layout_for_id(&fixture.project, &fixture.profile, project_id).unwrap(); + let mut meta = branch_meta::load_branch_meta(&layout.data_root).unwrap(); + let branches = layout.data_root.join("branches"); + fs::create_dir_all(&branches).unwrap(); + for index in 0..count { + let name = format!("load-{index:03}"); + let relative = format!("branches/load-{index:03}.db"); + fs::hard_link(&layout.graph_db_path, layout.data_root.join(&relative)).unwrap(); + meta.add_branch(&name, &relative, "main"); + } + branch_meta::save_branch_meta(&layout.data_root, &meta).unwrap(); +} + +fn sqlite_family_bytes(path: &Path) -> u64 { + [ + path.to_path_buf(), + sqlite_sidecar(path, "-wal"), + sqlite_sidecar(path, "-shm"), + ] + .into_iter() + .filter_map(|member| fs::metadata(member).ok()) + .map(|metadata| metadata.len()) + .sum() +} + +async fn execute_sql(path: &Path, sql: &str) { + let (db, _) = Database::open(path).await.unwrap(); + db.conn().execute_batch(sql).await.unwrap(); + db.checkpoint().await.unwrap(); + db.close(); +} + +fn init_repo(path: &Path) { + fs::create_dir_all(path).unwrap(); + run_git(path, &["init"]); + run_git(path, &["config", "user.email", "test@example.com"]); + run_git(path, &["config", "user.name", "TraceDecay Test"]); + fs::write(path.join("lib.rs"), "pub fn fixture() {}\n").unwrap(); + run_git(path, &["add", "."]); + run_git(path, &["commit", "-m", "fixture"]); +} + +fn run_git(path: &Path, args: &[&str]) { + let status = Command::new(crate::git::git_program()) + .args(args) + .current_dir(path) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); +} diff --git a/src/migrate/manifest.rs b/src/migrate/manifest.rs index 6bceb477..7d1cf5ee 100644 --- a/src/migrate/manifest.rs +++ b/src/migrate/manifest.rs @@ -3,7 +3,7 @@ use std::fs; use std::io; use std::path::{Component, Path, PathBuf}; -use libsql::{Builder, Connection, OpenFlags, Value}; +use libsql::{Connection, Value}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -1029,31 +1029,28 @@ fn verify_sqlite_artifact_contents(source: &Path, target: &Path) -> io::Result<( } async fn summarize_sqlite_database(path: &Path) -> io::Result { - let db = Builder::new_local(path) - .flags(OpenFlags::SQLITE_OPEN_READ_ONLY) - .build() - .await - .map_err(|e| { - invalid_manifest(&format!( - "failed to open SQLite DB '{}': {e}", - path.display() - )) - })?; - let conn = db.connect().map_err(|e| { + let snapshot = crate::sqlite_read_snapshot::open(path).await.map_err(|e| { invalid_manifest(&format!( - "failed to connect to SQLite DB '{}': {e}", + "failed to snapshot SQLite DB '{}': {e}", path.display() )) })?; - if !sqlite_quick_check(&conn, path).await? { + summarize_sqlite_connection(snapshot.connection(), path).await +} + +async fn summarize_sqlite_connection( + conn: &Connection, + path: &Path, +) -> io::Result { + if !sqlite_quick_check(conn, path).await? { return Err(invalid_manifest(&format!( "SQLite quick_check failed for '{}'", path.display() ))); } - let user_version = sqlite_i64(&conn, "PRAGMA user_version", path).await?; - let schema = sqlite_schema_summary(&conn, path).await?; - let tables = sqlite_table_summaries(&conn, path).await?; + let user_version = sqlite_i64(conn, "PRAGMA user_version", path).await?; + let schema = sqlite_schema_summary(conn, path).await?; + let tables = sqlite_table_summaries(conn, path).await?; Ok(SqliteLogicalSummary { user_version, schema, diff --git a/src/migrate/mod.rs b/src/migrate/mod.rs index 5c86a5ce..f9196196 100644 --- a/src/migrate/mod.rs +++ b/src/migrate/mod.rs @@ -1,3 +1,4 @@ +pub mod consolidate; pub mod hermes; pub mod inventory; pub mod manifest; diff --git a/src/open_store_holders.rs b/src/open_store_holders.rs new file mode 100644 index 00000000..18efda40 --- /dev/null +++ b/src/open_store_holders.rs @@ -0,0 +1,507 @@ +//! Read-only discovery of processes holding `TraceDecay` `SQLite` store files. + +use std::io; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OpenStoreHolder { + pub(crate) pid: u32, + pub(crate) command: String, + pub(crate) executable: Option, + pub(crate) version: Option, + pub(crate) paths: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum OpenStoreHolderScan { + Supported(Vec), + Unsupported { reason: String }, +} + +/// Finds processes that currently hold any member of the supplied `SQLite` +/// database families. The scan never signals or terminates a process. +pub(crate) fn scan(database_paths: &[PathBuf]) -> io::Result { + #[cfg(target_os = "linux")] + { + if !Path::new("/proc").is_dir() { + return Ok(OpenStoreHolderScan::Unsupported { + reason: "open-store process discovery requires a mounted Linux /proc filesystem" + .to_string(), + }); + } + scan_linux( + Path::new("/proc"), + database_paths, + std::process::id(), + probe_tracedecay_version, + ) + .map(OpenStoreHolderScan::Supported) + } + #[cfg(target_os = "macos")] + { + scan_macos(database_paths).map(OpenStoreHolderScan::Supported) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = database_paths; + Ok(OpenStoreHolderScan::Unsupported { + reason: format!( + "open-store process discovery is unavailable on {}", + std::env::consts::OS + ), + }) + } +} + +#[cfg(target_os = "macos")] +fn scan_macos(database_paths: &[PathBuf]) -> io::Result> { + use std::collections::BTreeMap; + use std::os::unix::fs::MetadataExt; + use std::process::Command; + + let mut targets = database_paths + .iter() + .flat_map(|path| sqlite_family_paths(path)) + .filter(|path| path.is_file()) + .map(|path| path.canonicalize().unwrap_or(path)) + .collect::>(); + targets.sort(); + targets.dedup(); + if targets.is_empty() { + return Ok(Vec::new()); + } + let mut identities = BTreeMap::<(u64, u64), Vec>::new(); + for target in &targets { + let metadata = target.metadata()?; + identities + .entry((metadata.dev(), metadata.ino())) + .or_default() + .push(target.clone()); + } + + let run = |program: &str| { + Command::new(program) + .args(["-nP", "-FpcfDi0", "--"]) + .args(&targets) + .output() + }; + let output = match run("lsof") { + Ok(output) => output, + Err(error) if error.kind() == io::ErrorKind::NotFound => run("/usr/sbin/lsof")?, + Err(error) => return Err(error), + }; + let stderr_has_content = output.stderr.iter().any(|byte| !byte.is_ascii_whitespace()); + if (!output.status.success() && output.status.code() != Some(1)) || stderr_has_content { + return Err(io::Error::other(format!( + "lsof failed while checking open TraceDecay stores: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + parse_lsof_output(&output.stdout, &identities, std::process::id()) +} + +#[cfg(any(target_os = "macos", all(test, unix)))] +fn parse_lsof_output( + output: &[u8], + targets: &std::collections::BTreeMap<(u64, u64), Vec>, + own_pid: u32, +) -> io::Result> { + use std::collections::BTreeSet; + + let mut holders = Vec::new(); + let mut pid = None; + let mut command = String::new(); + let mut paths = BTreeSet::new(); + let mut file_open = false; + let mut device = None; + let mut inode = None; + let finish_file = |file_open: &mut bool, + device: &mut Option, + inode: &mut Option, + paths: &mut BTreeSet| + -> io::Result<()> { + if !*file_open { + return Ok(()); + } + let identity = match (device.take(), inode.take()) { + (Some(device), Some(inode)) => (device, inode), + _ => { + return Err(io::Error::other( + "lsof returned a matching file without device and inode identity", + )); + } + }; + let Some(matched) = targets.get(&identity) else { + return Err(io::Error::other(format!( + "lsof returned unexpected file identity {:#x}:{}", + identity.0, identity.1 + ))); + }; + paths.extend(matched.iter().cloned()); + *file_open = false; + Ok(()) + }; + let finish = |pid: &mut Option, + command: &mut String, + paths: &mut BTreeSet, + holders: &mut Vec| { + let Some(current) = pid.take() else { + return; + }; + if current != own_pid && !paths.is_empty() { + holders.push(OpenStoreHolder { + pid: current, + command: std::mem::take(command), + executable: None, + version: None, + paths: std::mem::take(paths).into_iter().collect(), + }); + } else { + command.clear(); + paths.clear(); + } + }; + for field in output.split(|byte| *byte == 0) { + let field = field.strip_prefix(b"\n").unwrap_or(field); + let Some((&kind, value)) = field.split_first() else { + continue; + }; + match kind { + b'p' => { + finish_file(&mut file_open, &mut device, &mut inode, &mut paths)?; + finish(&mut pid, &mut command, &mut paths, &mut holders); + pid = Some( + parse_decimal_field(value) + .and_then(|value| u32::try_from(value).ok()) + .ok_or_else(|| io::Error::other("lsof returned an invalid process ID"))?, + ); + } + b'c' => command = String::from_utf8_lossy(value).into_owned(), + b'f' => { + if pid.is_none() { + return Err(io::Error::other( + "lsof returned a matching file without a process ID", + )); + } + finish_file(&mut file_open, &mut device, &mut inode, &mut paths)?; + file_open = true; + } + b'D' => device = parse_hex_field(value), + b'i' => inode = parse_decimal_field(value), + _ => {} + } + } + finish_file(&mut file_open, &mut device, &mut inode, &mut paths)?; + finish(&mut pid, &mut command, &mut paths, &mut holders); + holders.sort_by_key(|holder| holder.pid); + Ok(holders) +} + +#[cfg(any(target_os = "macos", all(test, unix)))] +fn parse_hex_field(value: &[u8]) -> Option { + let value = value.strip_prefix(b"0x").unwrap_or(value); + std::str::from_utf8(value) + .ok() + .and_then(|value| u64::from_str_radix(value, 16).ok()) +} + +#[cfg(any(target_os = "macos", all(test, unix)))] +fn parse_decimal_field(value: &[u8]) -> Option { + std::str::from_utf8(value) + .ok() + .and_then(|value| value.parse().ok()) +} + +#[cfg(target_os = "linux")] +fn scan_linux( + proc_root: &Path, + database_paths: &[PathBuf], + own_pid: u32, + mut version_probe: F, +) -> io::Result> +where + F: FnMut(u32, &Path, &str) -> Option, +{ + use std::collections::{BTreeMap, BTreeSet}; + use std::os::unix::fs::MetadataExt; + + let mut targets = BTreeMap::<(u64, u64), BTreeSet>::new(); + for database in database_paths { + for path in sqlite_family_paths(database) { + match std::fs::metadata(&path) { + Ok(metadata) if metadata.is_file() => { + targets + .entry((metadata.dev(), metadata.ino())) + .or_default() + .insert(path.canonicalize().unwrap_or(path)); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + } + if targets.is_empty() { + return Ok(Vec::new()); + } + + let mut holders = Vec::new(); + for entry in std::fs::read_dir(proc_root)? { + let entry = match entry { + Ok(entry) => entry, + Err(error) if transient_proc_error(&error) => continue, + Err(error) => return Err(error), + }; + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|value| value.parse::().ok()) + else { + continue; + }; + if pid == own_pid { + continue; + } + let process_root = entry.path(); + let fds = match std::fs::read_dir(process_root.join("fd")) { + Ok(fds) => fds, + Err(error) if transient_proc_error(&error) => continue, + Err(error) => return Err(error), + }; + let mut paths = BTreeSet::new(); + for fd in fds { + let fd = match fd { + Ok(fd) => fd, + Err(error) if transient_proc_error(&error) => continue, + Err(error) => return Err(error), + }; + let metadata = match std::fs::metadata(fd.path()) { + Ok(metadata) => metadata, + Err(error) if transient_proc_error(&error) => continue, + Err(error) => return Err(error), + }; + if let Some(matched) = targets.get(&(metadata.dev(), metadata.ino())) { + paths.extend(matched.iter().cloned()); + } + } + if paths.is_empty() { + continue; + } + + let command = process_comm(&process_root, pid); + let executable = std::fs::read_link(process_root.join("exe")).ok(); + let version = if is_tracedecay_process(&command, executable.as_deref()) { + version_probe(pid, proc_root, &command) + } else { + None + }; + holders.push(OpenStoreHolder { + pid, + command, + executable, + version, + paths: paths.into_iter().collect(), + }); + } + holders.sort_by_key(|holder| holder.pid); + Ok(holders) +} + +#[cfg(target_os = "linux")] +fn transient_proc_error(error: &io::Error) -> bool { + matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::PermissionDenied + ) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn sqlite_family_paths(path: &Path) -> [PathBuf; 3] { + [ + path.to_path_buf(), + with_suffix(path, "-wal"), + with_suffix(path, "-shm"), + ] +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut value = path.as_os_str().to_os_string(); + value.push(suffix); + PathBuf::from(value) +} + +#[cfg(all(test, unix))] +mod lsof_tests { + use super::*; + use std::collections::BTreeMap; + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + #[test] + fn lsof_field_output_is_bounded_to_targets_and_excludes_self() { + let target = PathBuf::from("/stores/sessions.db"); + let targets = BTreeMap::from([((0x2a, 7), vec![target.clone()])]); + let holders = parse_lsof_output( + b"p42\0ctracedecay\0f7\0D0x2a\0i7\0\np43\0cself\0f8\0D0x2a\0i7\0\n", + &targets, + 43, + ) + .unwrap(); + assert_eq!(holders.len(), 1); + assert_eq!(holders[0].pid, 42); + assert_eq!(holders[0].command, "tracedecay"); + assert_eq!(holders[0].paths, vec![target]); + } + + #[test] + fn lsof_field_output_preserves_non_utf8_and_newline_paths() { + let target = PathBuf::from(OsString::from_vec(b"/stores/odd\n\xff.db".to_vec())); + let targets = BTreeMap::from([((0x2a, 7), vec![target.clone()])]); + let output = b"p42\0ctracedecay\0f7\0D0x2a\0i7\0n/stores/odd\\n\\xff.db\0\n"; + + let holders = parse_lsof_output(output, &targets, 43).unwrap(); + assert_eq!(holders.len(), 1); + assert_eq!(holders[0].paths, vec![target]); + } + + #[test] + fn lsof_field_output_rejects_missing_file_identity() { + let targets = BTreeMap::from([((0x2a, 7), vec![PathBuf::from("/stores/db")])]); + let error = parse_lsof_output(b"p42\0ctracedecay\0f7\0\n", &targets, 43).unwrap_err(); + assert!(error.to_string().contains("without device and inode")); + } + + #[test] + fn lsof_field_output_rejects_missing_process_identity() { + let targets = BTreeMap::from([((0x2a, 7), vec![PathBuf::from("/stores/db")])]); + let error = parse_lsof_output(b"f7\0D0x2a\0i7\0\n", &targets, 43).unwrap_err(); + assert!(error.to_string().contains("without a process ID")); + } +} + +#[cfg(target_os = "linux")] +fn process_comm(process_root: &Path, pid: u32) -> String { + std::fs::read_to_string(process_root.join("comm")) + .map(|value| value.trim().to_string()) + .unwrap_or_else(|_| format!("pid {pid}")) +} + +#[cfg(target_os = "linux")] +fn is_tracedecay_process(command: &str, executable: Option<&Path>) -> bool { + let executable_matches = executable + .and_then(Path::file_name) + .is_some_and(|name| name.to_string_lossy().contains("tracedecay")); + let command_matches = command + .split_whitespace() + .next() + .and_then(|value| Path::new(value).file_name()) + .is_some_and(|name| name.to_string_lossy().contains("tracedecay")); + executable_matches || command_matches +} + +#[cfg(target_os = "linux")] +fn probe_tracedecay_version(pid: u32, proc_root: &Path, _command: &str) -> Option { + use std::process::{Command, Stdio}; + use std::thread; + use std::time::{Duration, Instant}; + + let mut child = Command::new(proc_root.join(pid.to_string()).join("exe")) + .arg("--version") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + let deadline = Instant::now() + Duration::from_secs(1); + loop { + match child.try_wait() { + Ok(Some(status)) if status.success() => { + let output = child.wait_with_output().ok()?; + return String::from_utf8(output.stdout) + .ok()? + .lines() + .next() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToOwned::to_owned); + } + Ok(Some(_)) | Err(_) => return None, + Ok(None) if Instant::now() < deadline => { + thread::sleep(Duration::from_millis(10)); + } + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + return None; + } + } + } +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use std::os::unix::fs::symlink; + + use tempfile::TempDir; + + use super::*; + + #[test] + fn linux_scan_matches_open_sidecars_by_file_identity() { + let temp = TempDir::new().unwrap(); + let proc_root = temp.path().join("proc"); + let database = temp.path().join("store/sessions.db"); + std::fs::create_dir_all(database.parent().unwrap()).unwrap(); + std::fs::write(&database, b"db").unwrap(); + let wal = with_suffix(&database, "-wal"); + std::fs::write(&wal, b"wal").unwrap(); + + let process = proc_root.join("42"); + std::fs::create_dir_all(process.join("fd")).unwrap(); + std::fs::write( + process.join("cmdline"), + b"/opt/tracedecay\0serve\0--token\0secret-value\0", + ) + .unwrap(); + std::fs::write(process.join("comm"), b"tracedecay\n").unwrap(); + symlink("/opt/tracedecay", process.join("exe")).unwrap(); + symlink(&wal, process.join("fd/7")).unwrap(); + + let holders = scan_linux(&proc_root, &[database], 9000, |pid, _, command| { + assert_eq!(pid, 42); + assert_eq!(command, "tracedecay"); + Some("tracedecay 0.0.45".to_string()) + }) + .unwrap(); + + assert_eq!(holders.len(), 1); + assert_eq!(holders[0].pid, 42); + assert_eq!(holders[0].version.as_deref(), Some("tracedecay 0.0.45")); + assert_eq!(holders[0].paths, vec![wal.canonicalize().unwrap()]); + assert!(!format!("{:?}", holders[0]).contains("secret-value")); + } + + #[test] + fn linux_scan_ignores_its_own_pid_and_unrelated_files() { + let temp = TempDir::new().unwrap(); + let proc_root = temp.path().join("proc"); + let database = temp.path().join("sessions.db"); + let unrelated = temp.path().join("other.db"); + std::fs::write(&database, b"db").unwrap(); + std::fs::write(&unrelated, b"other").unwrap(); + for (pid, path) in [(42_u32, &database), (43_u32, &unrelated)] { + let process = proc_root.join(pid.to_string()); + std::fs::create_dir_all(process.join("fd")).unwrap(); + std::fs::write(process.join("comm"), b"tracedecay\n").unwrap(); + symlink(path, process.join("fd/1")).unwrap(); + } + + let holders = scan_linux(&proc_root, &[database], 42, |_, _, _| { + panic!("excluded processes must not be probed") + }) + .unwrap(); + + assert!(holders.is_empty()); + } +} diff --git a/src/sqlite_read_snapshot.rs b/src/sqlite_read_snapshot.rs new file mode 100644 index 00000000..1e4ed37d --- /dev/null +++ b/src/sqlite_read_snapshot.rs @@ -0,0 +1,743 @@ +//! Side-effect-free logical inspection of `SQLite` database families. + +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::SystemTime; + +use fs2::FileExt; +use libsql::{Builder, Connection, OpenFlags}; +use sha2::{Digest, Sha256}; + +static NEXT_SNAPSHOT: AtomicU64 = AtomicU64::new(0); +const SQLITE_OPEN_URI: i32 = 0x0000_0040; + +pub(crate) struct SnapshotDatabase { + connection: Connection, + _database: libsql::Database, + source: PathBuf, + source_state: Vec, + path: PathBuf, + _scratch: Option>, + #[cfg(test)] + copied_bytes: u64, +} + +impl SnapshotDatabase { + pub(crate) fn connection(&self) -> &Connection { + &self.connection + } + + pub(crate) fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn validate_source(&self) -> io::Result<()> { + if family_state(&self.source)? == self.source_state { + return Ok(()); + } + Err(io::Error::other(format!( + "SQLite database family '{}' changed after its read snapshot", + self.source.display() + ))) + } + + pub(crate) fn source_generation(&self) -> SourceGeneration { + SourceGeneration { + source: self.source.clone(), + states: self.source_state.clone(), + } + } + + #[cfg(test)] + pub(crate) fn copied_bytes(&self) -> u64 { + self.copied_bytes + } +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceGeneration { + source: PathBuf, + states: Vec, +} + +impl SourceGeneration { + pub(crate) fn validate(&self) -> io::Result<()> { + if family_state(&self.source)? == self.states { + return Ok(()); + } + Err(io::Error::other(format!( + "SQLite database family '{}' changed after inspection", + self.source.display() + ))) + } +} + +pub(crate) struct SnapshotSet { + databases: BTreeMap, + copied_bytes: u64, + #[allow(dead_code)] + scratch: Arc, +} + +impl SnapshotSet { + pub(crate) async fn capture(paths: &[PathBuf]) -> io::Result { + let root = default_scratch_root(paths)?; + Self::capture_in(paths, &root).await + } + + pub(crate) async fn capture_in(paths: &[PathBuf], root: &Path) -> io::Result { + let scratch = Arc::new(create_scratch_directory(root, expected_owner(paths)?)?); + let mut unique = paths.to_vec(); + unique.sort(); + unique.dedup(); + let mut prepared = Vec::new(); + let mut copied_bytes = 0_u64; + for (index, path) in unique.into_iter().enumerate() { + let snapshot = prepare_one(&path, &scratch, index)?; + copied_bytes = copied_bytes.saturating_add(snapshot.copy_bytes); + prepared.push(snapshot); + } + let available = fs2::available_space(&scratch.path)?; + if copied_bytes > available { + return Err(io::Error::other(format!( + "insufficient scratch space for SQLite read snapshots: required {copied_bytes} bytes, available {available} bytes at '{}'", + scratch.path.display() + ))); + } + let mut databases = BTreeMap::new(); + for snapshot in prepared { + let source = snapshot.source.clone(); + let database = finish_one(snapshot, Arc::clone(&scratch)).await?; + databases.insert(source, database); + } + Ok(Self { + databases, + copied_bytes, + scratch, + }) + } + + pub(crate) fn get(&self, path: &Path) -> io::Result<&SnapshotDatabase> { + self.databases.get(path).ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("no frozen SQLite snapshot for '{}'", path.display()), + ) + }) + } + + pub(crate) fn validate_sources_unchanged(&self) -> io::Result<()> { + for database in self.databases.values() { + database.validate_source()?; + } + Ok(()) + } + + pub(crate) fn copied_bytes(&self) -> u64 { + self.copied_bytes + } + + #[cfg(test)] + pub(crate) fn database_count(&self) -> usize { + self.databases.len() + } +} + +struct PreparedSnapshot { + source: PathBuf, + source_state: Vec, + target: PathBuf, + mode: SnapshotMode, + copy_bytes: u64, +} + +#[derive(Clone, Copy)] +enum SnapshotMode { + DirectImmutable, + Reflink, + Copy, +} + +struct ScratchDirectory { + path: PathBuf, + _owner_lock: File, +} + +impl Drop for ScratchDirectory { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct FileState { + path: PathBuf, + bytes: u64, + modified: SystemTime, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, + #[cfg(unix)] + changed_seconds: i64, + #[cfg(unix)] + changed_nanoseconds: i64, + #[cfg(unix)] + links: u64, +} + +/// Opens one source family without mutating it. Checkpointed DBs are read +/// directly through `SQLite` immutable mode. WAL-backed DBs are reflinked when +/// supported, then fall back to one full copy with WAL/SHM copied alongside. +pub(crate) async fn open(path: &Path) -> io::Result { + let mut snapshots = SnapshotSet::capture(&[path.to_path_buf()]).await?; + snapshots.databases.remove(path).ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("no frozen SQLite snapshot for '{}'", path.display()), + ) + }) +} + +pub(crate) async fn open_in(path: &Path, root: &Path) -> io::Result { + let mut snapshots = SnapshotSet::capture_in(&[path.to_path_buf()], root).await?; + snapshots.databases.remove(path).ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("no frozen SQLite snapshot for '{}'", path.display()), + ) + }) +} + +pub(crate) fn family_fingerprint(path: &Path) -> io::Result { + use std::io::Read; + + let before = family_state(path)?; + let mut hash = Sha256::new(); + for (label, member) in [ + (b"db".as_slice(), path.to_path_buf()), + (b"wal", with_suffix(path, "-wal")), + ] { + if !member.is_file() { + continue; + } + let bytes = fs::metadata(&member)?.len(); + // BEGIN IMMEDIATE may create an empty WAL while acquiring the apply + // guard. An empty sidecar contains no logical database state. + if label == b"wal" && bytes == 0 { + continue; + } + hash.update(label); + hash.update(bytes.to_be_bytes()); + let mut file = fs::File::open(&member)?; + let mut buffer = vec![0_u8; 1024 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hash.update(&buffer[..read]); + } + } + if family_state(path)? != before { + return Err(changed_during_snapshot(path)); + } + Ok(hex::encode(hash.finalize())) +} + +fn prepare_one( + source: &Path, + scratch: &ScratchDirectory, + index: usize, +) -> io::Result { + let directory = scratch.path.join(index.to_string()); + create_private_directory(&directory)?; + let target = directory.join("database.db"); + let source_state = family_state(source)?; + let main = source_state + .iter() + .find(|state| state.path == source) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("SQLite database '{}' does not exist", source.display()), + ) + })?; + let has_wal = source_state + .iter() + .any(|state| state.path == with_suffix(source, "-wal")); + let mode = if has_wal { + if reflink_copy::reflink(source, &target).is_ok() { + SnapshotMode::Reflink + } else { + let _ = fs::remove_file(&target); + SnapshotMode::Copy + } + } else { + SnapshotMode::DirectImmutable + }; + let mut copy_bytes = if matches!(mode, SnapshotMode::Copy) { + main.bytes + } else { + 0 + }; + if !matches!(mode, SnapshotMode::DirectImmutable) { + for suffix in ["-wal", "-shm"] { + let source_member = with_suffix(source, suffix); + if let Some(state) = source_state + .iter() + .find(|state| state.path == source_member) + { + copy_bytes = copy_bytes.saturating_add(state.bytes); + } + } + } + if family_state(source)? != source_state { + return Err(changed_during_snapshot(source)); + } + Ok(PreparedSnapshot { + source: source.to_path_buf(), + source_state, + target, + mode, + copy_bytes, + }) +} + +async fn finish_one( + prepared: PreparedSnapshot, + scratch: Arc, +) -> io::Result { + if matches!(prepared.mode, SnapshotMode::Copy) { + fs::copy(&prepared.source, &prepared.target)?; + } + if !matches!(prepared.mode, SnapshotMode::DirectImmutable) { + for suffix in ["-wal", "-shm"] { + let source_member = with_suffix(&prepared.source, suffix); + let Some(_) = prepared + .source_state + .iter() + .find(|state| state.path == source_member) + else { + continue; + }; + fs::copy(&source_member, with_suffix(&prepared.target, suffix))?; + } + } + if family_state(&prepared.source)? != prepared.source_state { + return Err(changed_during_snapshot(&prepared.source)); + } + let (open_path, flags, scratch) = if matches!(prepared.mode, SnapshotMode::DirectImmutable) { + ( + PathBuf::from(immutable_uri(&prepared.source)?), + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::from_bits_retain(SQLITE_OPEN_URI), + None, + ) + } else { + ( + prepared.target.clone(), + OpenFlags::SQLITE_OPEN_READ_ONLY, + Some(scratch), + ) + }; + let database = Builder::new_local(&open_path) + .flags(flags) + .build() + .await + .map_err(io::Error::other)?; + let connection = database.connect().map_err(io::Error::other)?; + connection + .execute_batch("PRAGMA query_only = ON;") + .await + .map_err(io::Error::other)?; + let snapshot = SnapshotDatabase { + connection, + _database: database, + source: prepared.source, + source_state: prepared.source_state, + path: open_path, + _scratch: scratch, + #[cfg(test)] + copied_bytes: prepared.copy_bytes, + }; + snapshot.validate_source()?; + Ok(snapshot) +} + +fn changed_during_snapshot(source: &Path) -> io::Error { + io::Error::other(format!( + "SQLite database family '{}' changed while taking a read snapshot", + source.display() + )) +} + +fn create_scratch_directory( + root: &Path, + expected_uid: Option, +) -> io::Result { + ensure_private_root(root, expected_uid)?; + let cleanup_lock = open_private_lock(&root.join(".cleanup.lock"), true)?; + cleanup_lock.lock_exclusive()?; + cleanup_stale_directories(root)?; + for _ in 0..100 { + let id = NEXT_SNAPSHOT.fetch_add(1, Ordering::Relaxed); + let path = root.join(format!("read-{}-{id}", std::process::id())); + match create_private_directory(&path) { + Ok(()) => { + let owner_lock = open_private_lock(&path.join(".owner.lock"), true)?; + owner_lock.lock_exclusive()?; + FileExt::unlock(&cleanup_lock)?; + return Ok(ScratchDirectory { + path, + _owner_lock: owner_lock, + }); + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "could not allocate a unique SQLite read snapshot directory", + )) +} + +fn default_scratch_root(paths: &[PathBuf]) -> io::Result { + #[cfg(unix)] + { + let uid = expected_owner(paths)?.ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "no SQLite input path was supplied") + })?; + Ok(std::env::temp_dir().join(format!("tracedecay-sqlite-read-{uid}"))) + } + #[cfg(not(unix))] + { + let _ = paths; + Ok(std::env::temp_dir().join("tracedecay-sqlite-read")) + } +} + +fn expected_owner(paths: &[PathBuf]) -> io::Result> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let path = paths.first().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "no SQLite input path was supplied", + ) + })?; + Ok(Some(fs::metadata(path)?.uid())) + } + #[cfg(not(unix))] + { + let _ = paths; + Ok(None) + } +} + +fn ensure_private_root(root: &Path, expected_uid: Option) -> io::Result<()> { + match fs::symlink_metadata(root) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + return Err(io::Error::other(format!( + "SQLite scratch root '{}' is not a directory", + root.display() + ))); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + create_private_directory(root)?; + } + Err(error) => return Err(error), + } + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let metadata = fs::symlink_metadata(root)?; + if expected_uid.is_some_and(|uid| metadata.uid() != uid) { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "SQLite scratch root '{}' has the wrong owner", + root.display() + ), + )); + } + if metadata.permissions().mode() & 0o077 != 0 { + fs::set_permissions(root, fs::Permissions::from_mode(0o700))?; + } + } + Ok(()) +} + +fn create_private_directory(path: &Path) -> io::Result<()> { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(path) +} + +fn open_private_lock(path: &Path, create: bool) -> io::Result { + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .create(create) + .truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options.open(path) +} + +fn cleanup_stale_directories(root: &Path) -> io::Result<()> { + for entry in fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + if !name.to_string_lossy().starts_with("read-") { + continue; + } + let path = entry.path(); + if !fs::symlink_metadata(&path)?.is_dir() { + continue; + } + let removable = match open_private_lock(&path.join(".owner.lock"), false) { + Ok(lock) => lock.try_lock_exclusive().is_ok(), + Err(error) if error.kind() == io::ErrorKind::NotFound => true, + Err(error) => return Err(error), + }; + if removable { + fs::remove_dir_all(path)?; + } + } + Ok(()) +} + +fn family_state(path: &Path) -> io::Result> { + let mut states = Vec::new(); + for member in family_paths(path) { + match fs::metadata(&member) { + Ok(metadata) if metadata.is_file() => { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + states.push(FileState { + path: member, + bytes: metadata.len(), + modified: metadata.modified()?, + #[cfg(unix)] + device: metadata.dev(), + #[cfg(unix)] + inode: metadata.ino(), + #[cfg(unix)] + changed_seconds: metadata.ctime(), + #[cfg(unix)] + changed_nanoseconds: metadata.ctime_nsec(), + #[cfg(unix)] + links: metadata.nlink(), + }); + } + Ok(_) => { + return Err(io::Error::other(format!( + "SQLite family member '{}' is not a file", + member.display() + ))); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(states) +} + +fn family_paths(path: &Path) -> [PathBuf; 3] { + [ + path.to_path_buf(), + with_suffix(path, "-wal"), + with_suffix(path, "-shm"), + ] +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut value = path.as_os_str().to_os_string(); + value.push(suffix); + PathBuf::from(value) +} + +fn immutable_uri(path: &Path) -> io::Result { + let raw = path.to_str().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("SQLite path '{}' is not UTF-8", path.display()), + ) + })?; + let mut encoded = String::with_capacity(raw.len() + 24); + for ch in raw.chars() { + match ch { + '?' => encoded.push_str("%3f"), + '#' => encoded.push_str("%23"), + '%' => encoded.push_str("%25"), + other => encoded.push(other), + } + } + Ok(format!("file:{encoded}?immutable=1&mode=ro")) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[tokio::test] + async fn snapshot_reads_wal_rows_without_touching_source_bytes_or_mtime() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("source.db"); + let database = Builder::new_local(&path).build().await.unwrap(); + let connection = database.connect().unwrap(); + connection + .execute_batch( + "PRAGMA journal_mode=WAL; + CREATE TABLE durable(value TEXT NOT NULL); + INSERT INTO durable(value) VALUES ('wal-resident');", + ) + .await + .unwrap(); + assert!(with_suffix(&path, "-wal").metadata().unwrap().len() > 0); + let before = family_state(&path).unwrap(); + + let snapshot = open(&path).await.unwrap(); + let mut rows = snapshot + .connection() + .query("SELECT value FROM durable", ()) + .await + .unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(), + "wal-resident" + ); + assert_eq!(family_state(&path).unwrap(), before); + } + + #[tokio::test] + async fn checkpointed_database_reads_directly_without_copy_or_metadata_change() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("source.db"); + let database = Builder::new_local(&path).build().await.unwrap(); + let connection = database.connect().unwrap(); + connection + .execute_batch( + "CREATE TABLE durable(value TEXT NOT NULL); + INSERT INTO durable(value) VALUES ('checkpointed');", + ) + .await + .unwrap(); + drop(connection); + drop(database); + let before = family_state(&path).unwrap(); + let snapshots = SnapshotSet::capture(std::slice::from_ref(&path)) + .await + .unwrap(); + assert_eq!(snapshots.copied_bytes(), 0); + let mut rows = snapshots + .get(&path) + .unwrap() + .connection() + .query("SELECT value FROM durable", ()) + .await + .unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(), + "checkpointed" + ); + assert_eq!(family_state(&path).unwrap(), before); + } + + #[test] + fn empty_wal_does_not_change_the_content_fingerprint() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("source.db"); + fs::write(&path, b"database bytes").unwrap(); + let before = family_fingerprint(&path).unwrap(); + fs::write(with_suffix(&path, "-wal"), b"").unwrap(); + assert_eq!(family_fingerprint(&path).unwrap(), before); + fs::write(with_suffix(&path, "-wal"), b"logical frame").unwrap(); + assert_ne!(family_fingerprint(&path).unwrap(), before); + } + + #[cfg(unix)] + #[tokio::test] + async fn scratch_is_private_and_next_capture_cleans_crash_debris() { + use std::os::unix::fs::PermissionsExt; + + let temp = TempDir::new().unwrap(); + let path = temp.path().join("source.db"); + let scratch_root = temp.path().join("private-scratch"); + let database = Builder::new_local(&path).build().await.unwrap(); + database + .connect() + .unwrap() + .execute_batch("CREATE TABLE durable(value TEXT NOT NULL);") + .await + .unwrap(); + drop(database); + + ensure_private_root( + &scratch_root, + expected_owner(std::slice::from_ref(&path)).unwrap(), + ) + .unwrap(); + let stale = scratch_root.join("read-999999-0"); + create_private_directory(&stale).unwrap(); + fs::write(stale.join("database.db"), b"private session data").unwrap(); + fs::write(stale.join(".owner.lock"), b"").unwrap(); + + let snapshots = SnapshotSet::capture_in(&[path], &scratch_root) + .await + .unwrap(); + assert!( + !stale.exists(), + "an unlocked crashed snapshot must be cleaned" + ); + assert_eq!( + fs::metadata(&scratch_root).unwrap().permissions().mode() & 0o777, + 0o700 + ); + assert_eq!( + fs::metadata(&snapshots.scratch.path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + let live = snapshots.scratch.path.clone(); + drop(snapshots); + assert!( + !live.exists(), + "normal drop must remove copied database data" + ); + assert!( + fs::read_dir(&scratch_root) + .unwrap() + .all(|entry| entry.unwrap().file_name() == ".cleanup.lock") + ); + } +} diff --git a/src/tracedecay/lifecycle.rs b/src/tracedecay/lifecycle.rs index 747c2ad7..de1a0402 100644 --- a/src/tracedecay/lifecycle.rs +++ b/src/tracedecay/lifecycle.rs @@ -267,11 +267,13 @@ impl TraceDecay { } return Ok(Some(selected)); } + let command = + consolidation_dry_run_command(project_root, &candidate_inventory, &selected_inventory); Err(identity_cutover_conflict( project_root, &selected_inventory, &candidate_inventory, - "run an explicit backup-and-consolidate migration before changing the marker", + &format!("run the offline dry-run `{command}` before changing the marker"), )) } @@ -1114,6 +1116,23 @@ fn identity_cutover_conflict( } } +fn consolidation_dry_run_command( + project_root: &Path, + source: &StoreIdentityInventory, + target: &StoreIdentityInventory, +) -> String { + format!( + "tracedecay migrate consolidate --project {} --source-project-id {} --target-project-id {}", + shell_quote(&project_root.to_string_lossy()), + source.project_id, + target.project_id, + ) +} + +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\"'\"'")) +} + fn profile_relative(profile_root: &Path, path: &Path) -> Option { path.strip_prefix(profile_root) .ok() diff --git a/tests/core_cli_suite/cli_non_interactive_test.rs b/tests/core_cli_suite/cli_non_interactive_test.rs index 8a37127c..268dbd94 100644 --- a/tests/core_cli_suite/cli_non_interactive_test.rs +++ b/tests/core_cli_suite/cli_non_interactive_test.rs @@ -1152,7 +1152,15 @@ async fn status_surfaces_split_identity_conflict_without_suggesting_init() { assert!(stderr.contains("proj_status_selected"), "{stderr}"); assert!(stderr.contains("proj_status_legacy"), "{stderr}"); assert!( - stderr.contains("backup-and-consolidate migration"), + stderr.contains("tracedecay migrate consolidate"), + "{stderr}" + ); + assert!( + stderr.contains("--source-project-id proj_status_legacy"), + "{stderr}" + ); + assert!( + stderr.contains("--target-project-id proj_status_selected"), "{stderr}" ); assert!(!stderr.contains("run `tracedecay init`"), "{stderr}"); diff --git a/tests/storage_suite/storage_resolver_test.rs b/tests/storage_suite/storage_resolver_test.rs index eb4115e4..7b1e0d46 100644 --- a/tests/storage_suite/storage_resolver_test.rs +++ b/tests/storage_suite/storage_resolver_test.rs @@ -1174,6 +1174,13 @@ async fn corrupt_nonempty_cutover_store_reports_both_shards_without_switching() assert!(message.contains("sessions=1"), "{message}"); assert!(message.contains("facts=1"), "{message}"); assert!(message.contains("no files changed"), "{message}"); + assert!( + message.contains(&format!( + "tracedecay migrate consolidate --project '{}' --source-project-id {legacy_project_id} --target-project-id {cutover_project_id}", + project.display() + )), + "{message}" + ); assert_eq!( read_repository_identity_marker(&project) .unwrap()