diff --git a/crates/execution/assets/runners/wasi-module.js b/crates/execution/assets/runners/wasi-module.js index 7dfe4d0384..8bb1d01dc6 100644 --- a/crates/execution/assets/runners/wasi-module.js +++ b/crates/execution/assets/runners/wasi-module.js @@ -54,6 +54,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = const __agentOSWasiOpenDirectory = 2; const __agentOSWasiOpenExclusive = 4; const __agentOSWasiOpenTruncate = 8; + const __agentOSWasiFdflagsAppend = 1; const __agentOSWasiRightFdRead = 1n << 1n; const __agentOSWasiRightFdWrite = 1n << 6n; const __agentOSWasiDefaultRightsBase = 0xffffffffffffffffn; @@ -396,6 +397,14 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } } + _hasReadRights(rights) { + try { + return (BigInt(rights) & __agentOSWasiRightFdRead) !== 0n; + } catch { + return true; + } + } + _writeUint32(ptr, value) { try { this._memoryView().setUint32(Number(ptr) >>> 0, Number(value) >>> 0, true); @@ -779,6 +788,9 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } _mapFsError(error) { + __agentOSWasiDebug( + `fs error code=${String(error?.code ?? "")} message=${String(error?.message ?? error)}`, + ); switch (error?.code) { case "EACCES": case "EPERM": @@ -827,6 +839,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = refCount: 1, open: true, readOnly: entry.readOnly === true, + append: entry.append === true, }; } @@ -1380,19 +1393,28 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } } + const position = handle.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ) + : null; const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, bytes, 0, bytes.length, - null, + position, ) ); return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } if (handle?.kind === "guest-file" && typeof handle.targetFd === "number") { - const position = handle.append ? null : (handle.position ?? 0); + const position = handle.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ) + : (handle.position ?? 0); const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, @@ -1469,7 +1491,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } if (entry.kind === "file") { this._clearStatCache(); - const position = typeof entry.offset === "number" ? entry.offset : null; + const position = entry.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(entry.realFd).size ?? 0) + ) + : (typeof entry.offset === "number" ? entry.offset : null); const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( entry.realFd, @@ -1479,7 +1505,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = position, ) ); - if (typeof entry.offset === "number") { + if (entry.append) { + entry.offset = this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(entry.realFd).size ?? 0) + ); + } else if (typeof entry.offset === "number") { entry.offset += written; } return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); @@ -2206,8 +2236,12 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = this._clearStatCache(); } const fsConstants = __agentOSFs().constants ?? {}; + const requestedFdFlags = Number(_fdflags) >>> 0; + const append = (requestedFdFlags & __agentOSWasiFdflagsAppend) !== 0; let openFlags = requestedWriteAccess - ? fsConstants.O_RDWR ?? 2 + ? (this._hasReadRights(requestedRightsBase) + ? fsConstants.O_RDWR ?? 2 + : fsConstants.O_WRONLY ?? 1) : fsConstants.O_RDONLY ?? 0; if ((requestedFlags & __agentOSWasiOpenCreate) !== 0) { openFlags |= fsConstants.O_CREAT ?? 64; @@ -2218,6 +2252,9 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = if ((requestedFlags & __agentOSWasiOpenTruncate) !== 0) { openFlags |= fsConstants.O_TRUNC ?? 512; } + if (append) { + openFlags |= fsConstants.O_APPEND ?? 1024; + } if (openDirectory) { openFlags |= fsConstants.O_DIRECTORY ?? 0; } @@ -2235,10 +2272,13 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = hostPath: fsPath, readOnly: resolved.readOnly === true, realFd, - offset: 0, + offset: append + ? Number(__agentOSFs().fstatSync(realFd).size ?? 0) + : 0, + append, rightsBase: requestedRightsBase & allowedRightsInheriting, rightsInheriting: requestedRightsInheriting & allowedRightsInheriting, - fdFlags: (Number(_fdflags) >>> 0) & 0xffff, + fdFlags: requestedFdFlags & 0xffff, }); }); return this._measureWasiPhase("writeOpenedFd", () => this._writeUint32(openedFdPtr, openedFd)); diff --git a/crates/native-sidecar/src/execution.rs b/crates/native-sidecar/src/execution.rs index ef336720d7..050851ce80 100644 --- a/crates/native-sidecar/src/execution.rs +++ b/crates/native-sidecar/src/execution.rs @@ -6608,6 +6608,23 @@ where } let resolved = resolved; record_execute_phase("child_process_resolve_execution", phase_start.elapsed()); + let parent_sync_roots = { + let vm = self.vms.get(vm_id).ok_or_else(|| missing_vm_error(vm_id))?; + let parent = vm + .active_processes + .get(process_id) + .ok_or_else(|| missing_process_error(vm_id, process_id))?; + (parent.host_write_dirty_recursive() + || !parent.clean_host_writes_are_observable_recursive()) + .then(|| (parent.host_cwd.clone(), parent.guest_cwd.clone())) + }; + if let Some((host_cwd, guest_cwd)) = parent_sync_roots { + let vm = self + .vms + .get_mut(vm_id) + .ok_or_else(|| missing_vm_error(vm_id))?; + sync_process_host_roots_to_kernel(vm, &host_cwd, &guest_cwd)?; + } let (parent_kernel_pid, child_process_id) = { let vm = self .vms @@ -7111,6 +7128,29 @@ where } let resolved = resolved; record_execute_phase("child_process_resolve_execution", phase_start.elapsed()); + let parent_sync_roots = { + let vm = self.vms.get(vm_id).ok_or_else(|| missing_vm_error(vm_id))?; + let root = vm + .active_processes + .get(process_id) + .ok_or_else(|| missing_process_error(vm_id, process_id))?; + let parent = + Self::active_process_by_path(root, current_process_path).ok_or_else(|| { + SidecarError::InvalidState(format!( + "unknown child process path {current_process_label} during nested spawn" + )) + })?; + (parent.host_write_dirty_recursive() + || !parent.clean_host_writes_are_observable_recursive()) + .then(|| (parent.host_cwd.clone(), parent.guest_cwd.clone())) + }; + if let Some((host_cwd, guest_cwd)) = parent_sync_roots { + let vm = self + .vms + .get_mut(vm_id) + .ok_or_else(|| missing_vm_error(vm_id))?; + sync_process_host_roots_to_kernel(vm, &host_cwd, &guest_cwd)?; + } let sidecar_requests = self.sidecar_requests.clone(); let vm = self @@ -9837,6 +9877,14 @@ fn collect_process_host_sync_roots( fn sync_process_host_writes_to_kernel( vm: &mut VmState, process: &ActiveProcess, +) -> Result<(), SidecarError> { + sync_process_host_roots_to_kernel(vm, &process.host_cwd, &process.guest_cwd) +} + +fn sync_process_host_roots_to_kernel( + vm: &mut VmState, + process_host_cwd: &Path, + process_guest_cwd: &str, ) -> Result<(), SidecarError> { if vm.root_filesystem_mode != RootFilesystemMode::ReadOnly { let shadow_root = vm.cwd.clone(); @@ -9844,10 +9892,10 @@ fn sync_process_host_writes_to_kernel( } if !path_is_within_root( - &normalize_host_path(&process.host_cwd), + &normalize_host_path(process_host_cwd), &normalize_host_path(&vm.cwd), ) { - sync_host_directory_tree_to_kernel(vm, &process.host_cwd, &process.guest_cwd)?; + sync_host_directory_tree_to_kernel(vm, process_host_cwd, process_guest_cwd)?; } Ok(()) diff --git a/crates/native-sidecar/tests/service.rs b/crates/native-sidecar/tests/service.rs index 135256cab4..ec65dfedd2 100644 --- a/crates/native-sidecar/tests/service.rs +++ b/crates/native-sidecar/tests/service.rs @@ -1349,7 +1349,7 @@ ykAheWCsAteSEWVc0w==\n\ return fallback; } - let vendored = repo_root.join("packages/core/commands"); + let vendored = repo_root.join("packages/runtime-core/commands"); if vendored.exists() { let staged = temp_dir("agentos-native-sidecar-vendored-commands"); for command in ["bash", "cat", "mkdir", "printf", "sh"] { @@ -1372,6 +1372,29 @@ ykAheWCsAteSEWVc0w==\n\ return staged; } + let legacy_vendored = repo_root.join("packages/core/commands"); + if legacy_vendored.exists() { + let staged = temp_dir("agentos-native-sidecar-vendored-commands"); + for command in ["bash", "cat", "mkdir", "printf", "sh"] { + let source = legacy_vendored.join(command); + let target = staged.join(command); + fs::copy(&source, &target).unwrap_or_else(|error| { + panic!( + "copy vendored command {} -> {}: {error}", + source.display(), + target.display() + ) + }); + let mut permissions = fs::metadata(&target) + .expect("stat staged vendored command") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&target, permissions) + .expect("chmod staged vendored command"); + } + return staged; + } + panic!( "registry WASM commands are required for service fs regression tests: expected {}, {}, or {}", copied.display(), diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 61841d55a0..182ebb1049 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -138,14 +138,14 @@ so a reader sees the whole board at a glance. ## P0 — Runtime / VM correctness -### 1. brush-shell `>>` append truncates instead of appending +### 1. brush-shell `>>` append truncates instead of appending — DONE - **Broken:** `execSync` with `>>` onto a write-only file overwrites instead of appends. `expected 'changed' to be 'originalchanged'`. (issue: rivet-dev/agentos#1657) - **Objective:** `>>` opens `O_WRONLY|O_APPEND` against the kernel VFS and appends, identical to bash on Linux. -- **Proof:** `bridge-child-process.test.ts` "append redirection … succeeds like - Linux" passes un-skipped, plus a direct VFS append test. -- **rev:** `fix(runtime): honor >> append mode in guest shell VFS redirection` +- **Proof:** `bridge-child-process.test.ts` append redirection tests pass + un-skipped; direct kernel append and native sidecar append regressions pass. +- **rev:** `ouxrzutq` — `fix(runtime): honor >> append mode in guest shell VFS redirection` ### 2. brush-shell `cat < file` stdin redirection fails (exit 1) - **Broken:** `cat < stdin-input.txt` exits 1 — input redirection from a VFS path diff --git a/packages/runtime-browser/src/wasi-polyfill.ts b/packages/runtime-browser/src/wasi-polyfill.ts index 40ee39ab9f..c885a06d43 100644 --- a/packages/runtime-browser/src/wasi-polyfill.ts +++ b/packages/runtime-browser/src/wasi-polyfill.ts @@ -162,6 +162,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = const __agentOSWasiOpenDirectory = 2; const __agentOSWasiOpenExclusive = 4; const __agentOSWasiOpenTruncate = 8; + const __agentOSWasiFdflagsAppend = 1; const __agentOSWasiRightFdRead = 1n << 1n; const __agentOSWasiRightFdWrite = 1n << 6n; const __agentOSWasiDefaultRightsBase = 0xffffffffffffffffn; @@ -504,6 +505,14 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } } + _hasReadRights(rights) { + try { + return (BigInt(rights) & __agentOSWasiRightFdRead) !== 0n; + } catch { + return true; + } + } + _writeUint32(ptr, value) { try { this._memoryView().setUint32(Number(ptr) >>> 0, Number(value) >>> 0, true); @@ -887,6 +896,9 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } _mapFsError(error) { + __agentOSWasiDebug( + \`fs error code=\${String(error?.code ?? "")} message=\${String(error?.message ?? error)}\`, + ); switch (error?.code) { case "EACCES": case "EPERM": @@ -935,6 +947,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = refCount: 1, open: true, readOnly: entry.readOnly === true, + append: entry.append === true, }; } @@ -1488,19 +1501,28 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } } + const position = handle.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ) + : null; const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, bytes, 0, bytes.length, - null, + position, ) ); return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } if (handle?.kind === "guest-file" && typeof handle.targetFd === "number") { - const position = handle.append ? null : (handle.position ?? 0); + const position = handle.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ) + : (handle.position ?? 0); const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, @@ -1577,7 +1599,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } if (entry.kind === "file") { this._clearStatCache(); - const position = typeof entry.offset === "number" ? entry.offset : null; + const position = entry.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(entry.realFd).size ?? 0) + ) + : (typeof entry.offset === "number" ? entry.offset : null); const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( entry.realFd, @@ -1587,7 +1613,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = position, ) ); - if (typeof entry.offset === "number") { + if (entry.append) { + entry.offset = this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(entry.realFd).size ?? 0) + ); + } else if (typeof entry.offset === "number") { entry.offset += written; } return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); @@ -2314,8 +2344,12 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = this._clearStatCache(); } const fsConstants = __agentOSFs().constants ?? {}; + const requestedFdFlags = Number(_fdflags) >>> 0; + const append = (requestedFdFlags & __agentOSWasiFdflagsAppend) !== 0; let openFlags = requestedWriteAccess - ? fsConstants.O_RDWR ?? 2 + ? (this._hasReadRights(requestedRightsBase) + ? fsConstants.O_RDWR ?? 2 + : fsConstants.O_WRONLY ?? 1) : fsConstants.O_RDONLY ?? 0; if ((requestedFlags & __agentOSWasiOpenCreate) !== 0) { openFlags |= fsConstants.O_CREAT ?? 64; @@ -2326,6 +2360,9 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = if ((requestedFlags & __agentOSWasiOpenTruncate) !== 0) { openFlags |= fsConstants.O_TRUNC ?? 512; } + if (append) { + openFlags |= fsConstants.O_APPEND ?? 1024; + } if (openDirectory) { openFlags |= fsConstants.O_DIRECTORY ?? 0; } @@ -2343,10 +2380,13 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = hostPath: fsPath, readOnly: resolved.readOnly === true, realFd, - offset: 0, + offset: append + ? Number(__agentOSFs().fstatSync(realFd).size ?? 0) + : 0, + append, rightsBase: requestedRightsBase & allowedRightsInheriting, rightsInheriting: requestedRightsInheriting & allowedRightsInheriting, - fdFlags: (Number(_fdflags) >>> 0) & 0xffff, + fdFlags: requestedFdFlags & 0xffff, }); }); return this._measureWasiPhase("writeOpenedFd", () => this._writeUint32(openedFdPtr, openedFd)); diff --git a/packages/runtime-core/src/kernel-proxy.ts b/packages/runtime-core/src/kernel-proxy.ts index 2a0c5e30a3..90d014819b 100644 --- a/packages/runtime-core/src/kernel-proxy.ts +++ b/packages/runtime-core/src/kernel-proxy.ts @@ -2011,16 +2011,12 @@ export class NativeSidecarKernelProxy { return this.client.pread(this.session, this.vm, path, offset, length); }, pwrite: async (path, offset, data) => { - const bytes = - await this.createFilesystemView(includeLocalMounts).readFile(path); - const nextSize = Math.max(bytes.length, offset + data.length); - const updated = new Uint8Array(nextSize); - updated.set(bytes); - updated.set(data, offset); - await this.createFilesystemView(includeLocalMounts).writeFile( - path, - updated, - ); + const local = includeLocalMounts ? this.resolveLocalMount(path) : null; + if (local) { + this.assertLocalWritable(local.mount); + return local.mount.fs.pwrite(local.relativePath, offset, data); + } + return this.client.pwrite(this.session, this.vm, path, offset, data); }, }; } diff --git a/packages/runtime-core/src/sidecar-process.ts b/packages/runtime-core/src/sidecar-process.ts index 9a5e926945..d01bf4e2d9 100644 --- a/packages/runtime-core/src/sidecar-process.ts +++ b/packages/runtime-core/src/sidecar-process.ts @@ -890,6 +890,23 @@ export class SidecarProcess { }); } + async pwrite( + session: AuthenticatedSession, + vm: CreatedVm, + path: string, + offset: number, + content: Uint8Array, + ): Promise { + const encoded = encodeGuestFilesystemContent(content); + await this.guestFilesystemCall(session, vm, { + operation: "pwrite", + path, + offset, + content: encoded.content, + encoding: encoded.encoding, + }); + } + async mkdir( session: AuthenticatedSession, vm: CreatedVm, diff --git a/packages/runtime-core/src/test-runtime.ts b/packages/runtime-core/src/test-runtime.ts index aeb8c13260..5307c839a1 100644 --- a/packages/runtime-core/src/test-runtime.ts +++ b/packages/runtime-core/src/test-runtime.ts @@ -1213,7 +1213,10 @@ export class NodeFileSystem implements VirtualFileSystem { offset: number, data: Uint8Array, ): Promise { - const handle = await fs.open(this.normalizeTarget(targetPath), "r+"); + const handle = await fs.open( + this.normalizeTarget(targetPath), + fsSync.constants.O_WRONLY, + ); try { await handle.write(data, 0, data.length, offset); } finally { diff --git a/packages/runtime-core/tests/integration/bridge-child-process.test.ts b/packages/runtime-core/tests/integration/bridge-child-process.test.ts index e1298a2cc5..930256fccf 100644 --- a/packages/runtime-core/tests/integration/bridge-child-process.test.ts +++ b/packages/runtime-core/tests/integration/bridge-child-process.test.ts @@ -319,7 +319,17 @@ describeIf(!skipReason, 'bridge child_process → kernel routing', () => { fs.chmodSync('/tmp/write-only.txt', 0o200); // A real shell opens the append target write-only, so a 0o200 file is // appendable even though it cannot be read back until the chmod below. - execSync("printf changed >> /tmp/write-only.txt", { encoding: 'utf-8' }); + try { + execSync("printf changed >> /tmp/write-only.txt", { encoding: 'utf-8' }); + } catch (error) { + console.error(JSON.stringify({ + message: error instanceof Error ? error.message : String(error), + status: error && typeof error === 'object' && 'status' in error ? error.status : null, + stdout: error && typeof error === 'object' && 'stdout' in error ? String(error.stdout ?? '') : '', + stderr: error && typeof error === 'object' && 'stderr' in error ? String(error.stderr ?? '') : '' + })); + process.exit(99); + } fs.chmodSync('/tmp/write-only.txt', 0o600); console.log(JSON.stringify({ mode: 'loaded',