diff --git a/crates/execution/assets/runners/wasi-module.js b/crates/execution/assets/runners/wasi-module.js index b861b77ba9..769b0be992 100644 --- a/crates/execution/assets/runners/wasi-module.js +++ b/crates/execution/assets/runners/wasi-module.js @@ -40,6 +40,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = const __agentOSWasiErrnoNoent = 44; const __agentOSWasiErrnoNosys = 52; const __agentOSWasiErrnoNotdir = 54; + const __agentOSWasiErrnoNotempty = 55; const __agentOSWasiErrnoPipe = 64; const __agentOSWasiErrnoRofs = 69; const __agentOSWasiErrnoNotcapable = 76; @@ -799,6 +800,8 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return __agentOSWasiErrnoNoent; case "ENOTDIR": return __agentOSWasiErrnoNotdir; + case "ENOTEMPTY": + return __agentOSWasiErrnoNotempty; case "EEXIST": return __agentOSWasiErrnoExist; case "EINVAL": diff --git a/crates/native-sidecar/src/execution.rs b/crates/native-sidecar/src/execution.rs index 6448bb7583..68979f9430 100644 --- a/crates/native-sidecar/src/execution.rs +++ b/crates/native-sidecar/src/execution.rs @@ -9939,8 +9939,7 @@ pub(crate) fn sync_active_process_host_writes_to_kernel( vm: &mut VmState, ) -> Result<(), SidecarError> { if vm.root_filesystem_mode != RootFilesystemMode::ReadOnly { - let shadow_root = vm.cwd.clone(); - sync_host_directory_tree_to_kernel(vm, &shadow_root, "/")?; + sync_vm_shadow_root_to_kernel(vm)?; } let normalized_vm_root = normalize_host_path(&vm.cwd); @@ -9952,6 +9951,90 @@ pub(crate) fn sync_active_process_host_writes_to_kernel( Ok(()) } +/// Syncs the VM shadow root into the kernel VFS and reconciles deletions. +/// +/// The walk itself is additive (it copies shadow entries into the kernel), so +/// on its own it can never express a guest deletion performed directly on the +/// shadow tree — a removed file would be resurrected forever. To get real +/// Linux semantics the walk records every guest path it sees and diffs that +/// set against the previous walk's inventory: paths that were present in the +/// shadow before and are gone now are removed from the kernel VFS as well. +/// +/// Deletion propagation is scoped to the guest-writable shadow region: it only +/// runs for the VM shadow root walk (never for external host roots), re-checks +/// the protected/mount skip list, never touches the POSIX bootstrap skeleton, +/// and removes directories non-recursively so a kernel directory that still +/// has kernel-only content (for example a mount point or a path that was never +/// materialized into the shadow) is left in place with a warning. +fn sync_vm_shadow_root_to_kernel(vm: &mut VmState) -> Result<(), SidecarError> { + let shadow_root = normalize_host_path(&vm.cwd); + if host_sync_root_is_filesystem_root(&shadow_root) { + tracing::warn!("skipping host shadow sync rooted at the host filesystem root"); + return Ok(()); + } + let mut synced_file_times = BTreeMap::new(); + let mut inventory = BTreeSet::new(); + sync_host_directory_tree_to_kernel_inner( + vm, + &shadow_root, + &shadow_root, + "/", + &mut synced_file_times, + Some(&mut inventory), + )?; + propagate_shadow_deletions_to_kernel(vm, &inventory); + vm.shadow_sync_inventory = inventory; + Ok(()) +} + +/// Removes kernel paths whose shadow copy disappeared since the last walk. +/// +/// Best-effort by design: a failure to reconcile one stale path must not +/// poison the guest filesystem operation that triggered the sync, so failures +/// are surfaced as host-visible warnings instead of errors. Children sort +/// after their parents in the `BTreeSet`, so the reverse iteration removes +/// leaves before the directories that contain them. +fn propagate_shadow_deletions_to_kernel(vm: &mut VmState, current: &BTreeSet) { + if vm.shadow_sync_inventory.is_empty() { + return; + } + let stale: Vec = vm + .shadow_sync_inventory + .iter() + .rev() + .filter(|path| !current.contains(*path)) + .cloned() + .collect(); + for path in stale { + if path == "/" + || should_skip_shadow_sync_path(vm, &path) + || is_shadow_bootstrap_dir(&path) + { + continue; + } + let stat = match vm.kernel.lstat(&path) { + Ok(stat) => stat, + // Already gone (for example removed through the kernel-direct + // guest fs path, which mirrors deletions into the shadow itself). + Err(_) => continue, + }; + let result = if stat.is_directory && !stat.is_symbolic_link { + vm.kernel.remove_dir(&path) + } else { + vm.kernel.remove_file(&path) + }; + if let Err(error) = result { + if error.code() != "ENOENT" { + tracing::warn!( + path = %path, + error = %error, + "failed to propagate guest shadow deletion into the kernel VFS" + ); + } + } + } +} + fn collect_active_process_host_sync_roots( vm: &VmState, normalized_vm_root: &Path, @@ -9998,8 +10081,7 @@ fn sync_process_host_roots_to_kernel( process_guest_cwd: &str, ) -> Result<(), SidecarError> { if vm.root_filesystem_mode != RootFilesystemMode::ReadOnly { - let shadow_root = vm.cwd.clone(); - sync_host_directory_tree_to_kernel(vm, &shadow_root, "/")?; + sync_vm_shadow_root_to_kernel(vm)?; } if !path_is_within_root( @@ -10038,6 +10120,7 @@ fn sync_host_directory_tree_to_kernel( &normalized_host_root, &normalized_guest_root, &mut synced_file_times, + None, ) } @@ -10047,6 +10130,7 @@ fn sync_host_directory_tree_to_kernel_inner( current_host_dir: &Path, guest_root: &str, synced_file_times: &mut BTreeMap<(u64, u64), (u64, u64)>, + mut inventory: Option<&mut BTreeSet>, ) -> Result<(), SidecarError> { let entries = match fs::read_dir(current_host_dir) { Ok(entries) => entries, @@ -10108,6 +10192,15 @@ fn sync_host_directory_tree_to_kernel_inner( continue; } + // Record every shadow entry we can represent in the kernel (dirs, + // files, symlinks) so the deletion reconcile can tell "was in the + // shadow, now gone" apart from "never came from the shadow". + if let Some(inventory) = inventory.as_deref_mut() { + if file_type.is_dir() || file_type.is_file() || file_type.is_symlink() { + inventory.insert(guest_path.clone()); + } + } + if file_type.is_dir() { let metadata = entry.metadata().map_err(|error| { SidecarError::Io(format!( @@ -10143,6 +10236,7 @@ fn sync_host_directory_tree_to_kernel_inner( &host_path, guest_root, synced_file_times, + inventory.as_deref_mut(), )?; continue; } diff --git a/crates/native-sidecar/src/state.rs b/crates/native-sidecar/src/state.rs index 973d8534df..9cf857c76d 100644 --- a/crates/native-sidecar/src/state.rs +++ b/crates/native-sidecar/src/state.rs @@ -357,6 +357,15 @@ pub(crate) struct VmState { /// packages ship no `agentos-package.json`, so agent enumeration and /// resolution read this instead of the guest filesystem. pub(crate) projected_agent_launch: BTreeMap, + /// Guest paths that were present in the VM shadow root during the last + /// shadow->kernel sync walk. The next walk diffs the current shadow tree + /// against this set so guest deletions performed directly on the shadow + /// (host-side runtimes, WASI passthrough writes) propagate into the kernel + /// VFS instead of being resurrected by the otherwise additive sync. + /// Memory is bounded by the shadow tree itself, which is capped by the + /// kernel filesystem inode/byte resource limits that bound what the walk + /// can materialize. + pub(crate) shadow_sync_inventory: BTreeSet, } /// Launch parameters for one projected agent package. diff --git a/crates/native-sidecar/src/vm.rs b/crates/native-sidecar/src/vm.rs index cad03aa8c3..300ad28dc3 100644 --- a/crates/native-sidecar/src/vm.rs +++ b/crates/native-sidecar/src/vm.rs @@ -373,6 +373,7 @@ where signal_states: BTreeMap::new(), packages_staging_root: None, projected_agent_launch: BTreeMap::new(), + shadow_sync_inventory: BTreeSet::new(), }, ); diff --git a/crates/native-sidecar/tests/filesystem.rs b/crates/native-sidecar/tests/filesystem.rs index e414eff247..ad0329d229 100644 --- a/crates/native-sidecar/tests/filesystem.rs +++ b/crates/native-sidecar/tests/filesystem.rs @@ -463,6 +463,171 @@ mod shadow_root { } } + fn base_guest_filesystem_request( + operation: GuestFilesystemOperation, + path: &str, + ) -> GuestFilesystemCallRequest { + GuestFilesystemCallRequest { + operation, + path: String::from(path), + destination_path: None, + target: None, + content: None, + encoding: None, + recursive: false, + max_depth: None, + mode: None, + uid: None, + gid: None, + atime_ms: None, + mtime_ms: None, + len: None, + offset: None, + } + } + + fn guest_path_exists( + sidecar: &mut agentos_native_sidecar::NativeSidecar, + connection_id: &str, + session_id: &str, + vm_id: &str, + request_id: i64, + path: &str, + ) -> bool { + let response = sidecar + .dispatch_wire_blocking(wire_request( + request_id, + wire_vm(connection_id, session_id, vm_id), + RequestPayload::GuestFilesystemCallRequest(base_guest_filesystem_request( + GuestFilesystemOperation::Exists, + path, + )), + )) + .expect("dispatch guest filesystem exists"); + match response.response.payload { + ResponsePayload::GuestFilesystemResultResponse(result) => { + result.exists.unwrap_or(false) + } + other => panic!("expected guest_filesystem_result response, got {other:?}"), + } + } + + /// Deleting a path directly from the VM shadow root (the way host-side + /// guest runtimes delete files, without a kernel-direct unlink) must + /// propagate into the kernel VFS on the next shadow sync walk instead of + /// being resurrected by the additive copy-in. + #[test] + fn shadow_direct_deletions_propagate_into_kernel_vfs() { + let mut sidecar = create_test_sidecar(); + let (connection_id, session_id) = authenticate_and_open_session(&mut sidecar); + // Guest filesystem calls need no command mounts; create the VM bare so + // this regression test runs even without built registry commands. + let cwd = temp_dir("filesystem-shadow-reconcile-cwd"); + let (vm_id, _) = create_vm_wire( + &mut sidecar, + 3, + &connection_id, + &session_id, + GuestRuntimeKind::JavaScript, + &cwd, + ); + + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_nanos(); + let doomed_root = format!("/workspace/reconcile-{nonce}"); + let doomed_dir = format!("{doomed_root}/nested"); + let doomed_file = format!("{doomed_dir}/probe.txt"); + let survivor_file = format!("/workspace/reconcile-survivor-{nonce}.txt"); + + let mut mkdir = + base_guest_filesystem_request(GuestFilesystemOperation::CreateDir, &doomed_dir); + mkdir.recursive = true; + guest_filesystem_call(&mut sidecar, &connection_id, &session_id, &vm_id, 20, mkdir); + + let mut write = + base_guest_filesystem_request(GuestFilesystemOperation::WriteFile, &doomed_file); + write.content = Some(String::from("doomed\n")); + write.encoding = Some(RootFilesystemEntryEncoding::Utf8); + guest_filesystem_call(&mut sidecar, &connection_id, &session_id, &vm_id, 21, write); + + let mut survivor = + base_guest_filesystem_request(GuestFilesystemOperation::WriteFile, &survivor_file); + survivor.content = Some(String::from("survivor\n")); + survivor.encoding = Some(RootFilesystemEntryEncoding::Utf8); + guest_filesystem_call(&mut sidecar, &connection_id, &session_id, &vm_id, 22, survivor); + + // This read-side call walks the shadow tree and records the inventory + // the deletion reconcile diffs against. + assert!(guest_path_exists( + &mut sidecar, + &connection_id, + &session_id, + &vm_id, + 23, + &doomed_file, + )); + + // Locate this VM's shadow root through the mirrored unique file, then + // delete the subtree exactly as a host-side guest runtime would: on + // the shadow filesystem only, with no kernel-direct unlink. + let marker_rel = format!( + "workspace/reconcile-{nonce}/nested/probe.txt" + ); + let shadow_root = std::fs::read_dir(std::env::temp_dir()) + .expect("list temp dir") + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .find(|candidate| { + candidate + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("agentos-native-sidecar-shadow-")) + && candidate.join(&marker_rel).is_file() + }) + .expect("locate VM shadow root through mirrored probe file"); + fs::remove_dir_all(shadow_root.join(format!("workspace/reconcile-{nonce}"))) + .expect("delete subtree from the shadow root"); + + // The next host filesystem call re-walks the shadow; the kernel must + // drop the deleted subtree instead of resurrecting it forever. + assert!( + !guest_path_exists( + &mut sidecar, + &connection_id, + &session_id, + &vm_id, + 24, + &doomed_file, + ), + "kernel resurrected a file deleted from the shadow root" + ); + assert!( + !guest_path_exists( + &mut sidecar, + &connection_id, + &session_id, + &vm_id, + 25, + &doomed_root, + ), + "kernel resurrected a directory deleted from the shadow root" + ); + assert!( + guest_path_exists( + &mut sidecar, + &connection_id, + &session_id, + &vm_id, + 26, + &survivor_file, + ), + "deletion reconcile must not remove shadow-backed paths that still exist" + ); + + dispose_vm_and_close_session(&mut sidecar, &connection_id, &session_id, &vm_id); + } + #[allow(clippy::too_many_arguments)] fn execute_command( sidecar: &mut agentos_native_sidecar::NativeSidecar, diff --git a/crates/vfs/src/posix/overlay_fs.rs b/crates/vfs/src/posix/overlay_fs.rs index 9fd0600587..653d18c819 100644 --- a/crates/vfs/src/posix/overlay_fs.rs +++ b/crates/vfs/src/posix/overlay_fs.rs @@ -1563,8 +1563,13 @@ impl VirtualFileSystem for OverlayFileSystem { if self.is_whited_out(path) { return Err(Self::entry_not_found(path)); } - let lower_exists = self.find_lower_by_exists(path).is_some(); - let upper_exists = self.exists_in_upper(path); + // POSIX unlink(2) removes the directory entry itself and never follows + // a symlink leaf, so existence must be checked with lstat semantics. + // `exists()` resolves symlinks, which made dangling symlinks (for + // example git's `.git/tXXXXXX` symlink-support probe) unremovable: + // the entry stayed listed by readdir while unlink reported ENOENT. + let lower_exists = self.find_lower_by_entry(path).is_some(); + let upper_exists = self.has_entry_in_upper(path); if !lower_exists && !upper_exists { return Err(Self::entry_not_found(path)); } @@ -2013,6 +2018,55 @@ mod tests { assert!(!overlay.exists("/workspace-temp")); } + #[test] + fn remove_file_unlinks_dangling_symlink() { + // git's symlink-support probe creates `.git/tXXXXXX -> testing` (a + // dangling symlink), lstats it, and unlinks it. unlink(2) must remove + // the link itself without resolving it. + let lower = MemoryFileSystem::new(); + let mut overlay = OverlayFileSystem::new(vec![lower], OverlayMode::Ephemeral); + overlay.mkdir("/repo", true).expect("create directory"); + overlay + .symlink("testing", "/repo/probe") + .expect("create dangling symlink"); + assert!(overlay + .lstat("/repo/probe") + .expect("lstat dangling symlink") + .is_symbolic_link); + + overlay + .remove_file("/repo/probe") + .expect("unlink dangling symlink"); + + assert!(overlay.lstat("/repo/probe").is_err()); + assert_eq!( + overlay.read_dir("/repo").expect("read emptied directory"), + Vec::::new() + ); + overlay + .remove_dir("/repo") + .expect("rmdir emptied directory"); + } + + #[test] + fn remove_file_unlinks_dangling_symlink_from_lower_layer() { + let mut lower = MemoryFileSystem::new(); + lower.mkdir("/repo", true).expect("create lower directory"); + lower + .symlink("missing-target", "/repo/probe") + .expect("seed lower dangling symlink"); + + let mut overlay = OverlayFileSystem::new(vec![lower], OverlayMode::Ephemeral); + overlay + .remove_file("/repo/probe") + .expect("whiteout lower dangling symlink"); + + assert!(overlay.lstat("/repo/probe").is_err()); + overlay + .remove_dir("/repo") + .expect("rmdir merged-empty directory"); + } + #[test] fn remove_dir_still_rejects_visible_children() { let mut lower = MemoryFileSystem::new(); diff --git a/packages/runtime-core/tests/integration/vfs-consistency.test.ts b/packages/runtime-core/tests/integration/vfs-consistency.test.ts index 19939a6ff8..128ee05196 100644 --- a/packages/runtime-core/tests/integration/vfs-consistency.test.ts +++ b/packages/runtime-core/tests/integration/vfs-consistency.test.ts @@ -109,4 +109,67 @@ describeIf(!skipReason, 'cross-runtime VFS consistency', () => { ); expect(nodeResult.exitCode).not.toBe(0); }); + + // Guest deletions must propagate into the kernel VFS instead of being + // resurrected by the (otherwise additive) shadow->kernel sync. This is the + // failure mode behind git's failed-clone junk surviving `remove_junk`. + it('guest rm -r removes the tree from the kernel VFS', async () => { + ctx = await createIntegrationKernel(); + + const create = await ctx.kernel.exec( + 'sh -c "mkdir -p /tmp/doomed/nested && echo payload > /tmp/doomed/nested/file.txt"', + ); + expect(create.exitCode).toBe(0); + expect(await ctx.vfs.exists('/tmp/doomed/nested/file.txt')).toBe(true); + + const remove = await ctx.kernel.exec('rm -r /tmp/doomed'); + expect(remove.exitCode).toBe(0); + + expect(await ctx.vfs.exists('/tmp/doomed/nested/file.txt')).toBe(false); + expect(await ctx.vfs.exists('/tmp/doomed/nested')).toBe(false); + expect(await ctx.vfs.exists('/tmp/doomed')).toBe(false); + + const ls = await ctx.kernel.exec('ls /tmp/doomed'); + expect(ls.exitCode).not.toBe(0); + }); + + it('guest rmdir on an empty directory succeeds and propagates', async () => { + ctx = await createIntegrationKernel(); + + const mkdir = await ctx.kernel.exec('mkdir /tmp/empty-dir'); + expect(mkdir.exitCode).toBe(0); + expect(await ctx.vfs.exists('/tmp/empty-dir')).toBe(true); + + const rmdir = await ctx.kernel.exec('rmdir /tmp/empty-dir'); + expect(rmdir.exitCode, rmdir.stderr).toBe(0); + expect(await ctx.vfs.exists('/tmp/empty-dir')).toBe(false); + }); + + it('guest unlink removes a dangling symlink so its directory can be emptied', async () => { + // git's symlink-support probe: create `tXXXXXX -> testing` (dangling), + // lstat it, unlink it, then remove the directory. unlink(2) must not + // resolve the symlink leaf, and rmdir must not report EIO afterwards. + ctx = await createIntegrationKernel(); + + const probe = await ctx.kernel.exec( + 'sh -c "mkdir /tmp/probe-dir && ln -s missing-target /tmp/probe-dir/probe && rm /tmp/probe-dir/probe && rmdir /tmp/probe-dir"', + ); + expect(probe.exitCode, probe.stderr).toBe(0); + expect(await ctx.vfs.exists('/tmp/probe-dir')).toBe(false); + }); + + it('guest rmdir on a non-empty directory reports ENOTEMPTY, not EIO', async () => { + ctx = await createIntegrationKernel(); + + const setup = await ctx.kernel.exec( + 'sh -c "mkdir /tmp/full-dir && echo keep > /tmp/full-dir/keep.txt"', + ); + expect(setup.exitCode).toBe(0); + + const rmdir = await ctx.kernel.exec('rmdir /tmp/full-dir'); + expect(rmdir.exitCode).not.toBe(0); + expect(rmdir.stderr.toLowerCase()).toContain('not empty'); + expect(rmdir.stderr.toLowerCase()).not.toContain('i/o error'); + expect(await ctx.vfs.exists('/tmp/full-dir/keep.txt')).toBe(true); + }); }); diff --git a/toolchain/c/build/cat b/toolchain/c/build/cat deleted file mode 100644 index f59943b868..0000000000 Binary files a/toolchain/c/build/cat and /dev/null differ diff --git a/toolchain/c/build/env b/toolchain/c/build/env deleted file mode 100644 index 401e013931..0000000000 Binary files a/toolchain/c/build/env and /dev/null differ diff --git a/toolchain/c/build/sort b/toolchain/c/build/sort deleted file mode 100644 index 95605b0e06..0000000000 Binary files a/toolchain/c/build/sort and /dev/null differ diff --git a/toolchain/c/build/wc b/toolchain/c/build/wc deleted file mode 100644 index 40bcb6891d..0000000000 Binary files a/toolchain/c/build/wc and /dev/null differ diff --git a/toolchain/test-programs/cat.c b/toolchain/test-programs/c-cat.c similarity index 100% rename from toolchain/test-programs/cat.c rename to toolchain/test-programs/c-cat.c diff --git a/toolchain/test-programs/env.c b/toolchain/test-programs/c-env.c similarity index 100% rename from toolchain/test-programs/env.c rename to toolchain/test-programs/c-env.c diff --git a/toolchain/test-programs/sort.c b/toolchain/test-programs/c-sort.c similarity index 100% rename from toolchain/test-programs/sort.c rename to toolchain/test-programs/c-sort.c diff --git a/toolchain/test-programs/wc.c b/toolchain/test-programs/c-wc.c similarity index 100% rename from toolchain/test-programs/wc.c rename to toolchain/test-programs/c-wc.c